1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValueUtil;
37 import org.apache.hadoop.hbase.testclassification.SmallTests;
38 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
39 import org.apache.hadoop.hbase.io.hfile.HFile;
40 import org.apache.hadoop.hbase.io.hfile.HFileContext;
41 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
42 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.AfterClass;
45 import org.junit.BeforeClass;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49 @Category(SmallTests.class)
50 public class TestHalfStoreFileReader {
51 private static HBaseTestingUtility TEST_UTIL;
52
53 @BeforeClass
54 public static void setupBeforeClass() throws Exception {
55 TEST_UTIL = new HBaseTestingUtility();
56 }
57
58 @AfterClass
59 public static void tearDownAfterClass() throws Exception {
60 TEST_UTIL.cleanupTestDir();
61 }
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 @Test
80 public void testHalfScanAndReseek() throws IOException {
81 String root_dir = TEST_UTIL.getDataTestDir().toString();
82 Path p = new Path(root_dir, "test");
83
84 Configuration conf = TEST_UTIL.getConfiguration();
85 FileSystem fs = FileSystem.get(conf);
86 CacheConfig cacheConf = new CacheConfig(conf);
87 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
88 HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
89 .withPath(fs, p)
90 .withFileContext(meta)
91 .create();
92
93
94 List<KeyValue> items = genSomeKeys();
95 for (KeyValue kv : items) {
96 w.append(kv);
97 }
98 w.close();
99
100 HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
101 r.loadFileInfo();
102 byte [] midkey = r.midkey();
103 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
104 midkey = midKV.getRow();
105
106
107
108 Reference bottom = new Reference(midkey, Reference.Range.bottom);
109 doTestOfScanAndReseek(p, fs, bottom, cacheConf);
110
111 Reference top = new Reference(midkey, Reference.Range.top);
112 doTestOfScanAndReseek(p, fs, top, cacheConf);
113
114 r.close();
115 }
116
117 private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
118 CacheConfig cacheConf)
119 throws IOException {
120 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
121 cacheConf, bottom, TEST_UTIL.getConfiguration());
122 halfreader.loadFileInfo();
123 final HFileScanner scanner = halfreader.getScanner(false, false);
124
125 scanner.seekTo();
126 Cell curr;
127 do {
128 curr = scanner.getKeyValue();
129 KeyValue reseekKv =
130 getLastOnCol(curr);
131 int ret = scanner.reseekTo(reseekKv);
132 assertTrue("reseek to returned: " + ret, ret > 0);
133
134 } while (scanner.next());
135
136 int ret = scanner.reseekTo(getLastOnCol(curr));
137
138 assertTrue( ret > 0 );
139
140 halfreader.close(true);
141 }
142
143
144
145 @Test
146 public void testHalfScanner() throws IOException {
147 String root_dir = TEST_UTIL.getDataTestDir().toString();
148 Path p = new Path(root_dir, "test");
149 Configuration conf = TEST_UTIL.getConfiguration();
150 FileSystem fs = FileSystem.get(conf);
151 CacheConfig cacheConf = new CacheConfig(conf);
152 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
153 HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
154 .withPath(fs, p)
155 .withFileContext(meta)
156 .create();
157
158
159 List<KeyValue> items = genSomeKeys();
160 for (KeyValue kv : items) {
161 w.append(kv);
162 }
163 w.close();
164
165
166 HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
167 r.loadFileInfo();
168 byte[] midkey = r.midkey();
169 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
170 midkey = midKV.getRow();
171
172 Reference bottom = new Reference(midkey, Reference.Range.bottom);
173 Reference top = new Reference(midkey, Reference.Range.top);
174
175
176 KeyValue beforeMidKey = null;
177 for (KeyValue item : items) {
178 if (KeyValue.COMPARATOR.compare(item, midKV) >= 0) {
179 break;
180 }
181 beforeMidKey = item;
182 }
183 System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
184 System.out.println("beforeMidKey: " + beforeMidKey);
185
186
187
188 Cell foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
189 assertEquals(beforeMidKey, foundKeyValue);
190
191
192 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
193 assertEquals(items.get(items.size() - 2), foundKeyValue);
194
195 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
196 assertEquals(beforeMidKey, foundKeyValue);
197
198
199 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
200 assertNull(foundKeyValue);
201
202
203 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
204 assertNull(foundKeyValue);
205
206
207 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
208 assertNull(foundKeyValue);
209
210 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
211 assertEquals(items.get(0), foundKeyValue);
212
213
214 foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
215 assertNull(foundKeyValue);
216 }
217
218 private Cell doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
219 CacheConfig cacheConfig)
220 throws IOException {
221 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
222 cacheConfig, bottom, TEST_UTIL.getConfiguration());
223 halfreader.loadFileInfo();
224 final HFileScanner scanner = halfreader.getScanner(false, false);
225 scanner.seekBefore(seekBefore);
226 return scanner.getKeyValue();
227 }
228
229 private KeyValue getLastOnCol(Cell curr) {
230 return KeyValueUtil.createLastOnRow(
231 curr.getRowArray(), curr.getRowOffset(), curr.getRowLength(),
232 curr.getFamilyArray(), curr.getFamilyOffset(), curr.getFamilyLength(),
233 curr.getQualifierArray(), curr.getQualifierOffset(), curr.getQualifierLength());
234 }
235
236 static final int SIZE = 1000;
237
238 static byte[] _b(String s) {
239 return Bytes.toBytes(s);
240 }
241
242 List<KeyValue> genSomeKeys() {
243 List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
244 for (int i = 0; i < SIZE; i++) {
245 KeyValue kv =
246 new KeyValue(
247 _b(String.format("row_%04d", i)),
248 _b("family"),
249 _b("qualifier"),
250 1000,
251 _b("value"));
252 ret.add(kv);
253 }
254 return ret;
255 }
256
257
258
259 }
260