View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Map;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestCase;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValue.Type;
40  import org.apache.hadoop.hbase.testclassification.SmallTests;
41  import org.apache.hadoop.hbase.Tag;
42  import org.apache.hadoop.hbase.io.compress.Compression;
43  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
44  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.io.Writable;
47  import org.junit.experimental.categories.Category;
48  
49  /**
50   * test hfile features.
51   * <p>
52   * Copied from
53   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
54   * Remove after tfile is committed and use the tfile version of this class
55   * instead.</p>
56   */
57  @Category(SmallTests.class)
58  public class TestHFile extends HBaseTestCase {
59    private static final Log LOG = LogFactory.getLog(TestHFile.class);
60  
61    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static String ROOT_DIR =
63      TEST_UTIL.getDataTestDir("TestHFile").toString();
64    private final int minBlockSize = 512;
65    private static String localFormatter = "%010d";
66    private static CacheConfig cacheConf = null;
67    private Map<String, Long> startingMetrics;
68  
69    @Override
70    public void setUp() throws Exception {
71      super.setUp();
72    }
73  
74    @Override
75    public void tearDown() throws Exception {
76      super.tearDown();
77    }
78  
79  
80    /**
81     * Test empty HFile.
82     * Test all features work reasonably when hfile is empty of entries.
83     * @throws IOException
84     */
85    public void testEmptyHFile() throws IOException {
86      if (cacheConf == null) cacheConf = new CacheConfig(conf);
87      Path f = new Path(ROOT_DIR, getName());
88      HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
89      Writer w =
90          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
91      w.close();
92      Reader r = HFile.createReader(fs, f, cacheConf, conf);
93      r.loadFileInfo();
94      assertNull(r.getFirstKey());
95      assertNull(r.getLastKey());
96    }
97  
98    /**
99     * Create 0-length hfile and show that it fails
100    */
101   public void testCorrupt0LengthHFile() throws IOException {
102     if (cacheConf == null) cacheConf = new CacheConfig(conf);
103     Path f = new Path(ROOT_DIR, getName());
104     FSDataOutputStream fsos = fs.create(f);
105     fsos.close();
106 
107     try {
108       Reader r = HFile.createReader(fs, f, cacheConf, conf);
109     } catch (CorruptHFileException che) {
110       // Expected failure
111       return;
112     }
113     fail("Should have thrown exception");
114   }
115 
116   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
117     FileStatus fst = fs.getFileStatus(src);
118     long len = fst.getLen();
119     len = len / 2 ;
120 
121     // create a truncated hfile
122     FSDataOutputStream fdos = fs.create(dst);
123     byte[] buf = new byte[(int)len];
124     FSDataInputStream fdis = fs.open(src);
125     fdis.read(buf);
126     fdos.write(buf);
127     fdis.close();
128     fdos.close();
129   }
130 
131   /**
132    * Create a truncated hfile and verify that exception thrown.
133    */
134   public void testCorruptTruncatedHFile() throws IOException {
135     if (cacheConf == null) cacheConf = new CacheConfig(conf);
136     Path f = new Path(ROOT_DIR, getName());
137     HFileContext  context = new HFileContextBuilder().build();
138     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
139         .withFileContext(context).create();
140     writeSomeRecords(w, 0, 100, false);
141     w.close();
142 
143     Path trunc = new Path(f.getParent(), "trucated");
144     truncateFile(fs, w.getPath(), trunc);
145 
146     try {
147       Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
148     } catch (CorruptHFileException che) {
149       // Expected failure
150       return;
151     }
152     fail("Should have thrown exception");
153   }
154 
155   // write some records into the tfile
156   // write them twice
157   private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
158       throws IOException {
159     String value = "value";
160     KeyValue kv;
161     for (int i = start; i < (start + n); i++) {
162       String key = String.format(localFormatter, Integer.valueOf(i));
163       if (useTags) {
164         Tag t = new Tag((byte) 1, "myTag1");
165         Tag[] tags = new Tag[1];
166         tags[0] = t;
167         kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
168             HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
169         writer.append(kv);
170       } else {
171         kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
172             Bytes.toBytes(value + key));
173         writer.append(kv);
174       }
175     }
176     return (start + n);
177   }
178 
179   private void readAllRecords(HFileScanner scanner) throws IOException {
180     readAndCheckbytes(scanner, 0, 100);
181   }
182 
183   // read the records and check
184   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
185       throws IOException {
186     String value = "value";
187     int i = start;
188     for (; i < (start + n); i++) {
189       ByteBuffer key = scanner.getKey();
190       ByteBuffer val = scanner.getValue();
191       String keyStr = String.format(localFormatter, Integer.valueOf(i));
192       String valStr = value + keyStr;
193       KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
194           Bytes.toBytes("qual"), Bytes.toBytes(valStr));
195       byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
196           Bytes.toBytes(key).length).getKey();
197       assertTrue("bytes for keys do not match " + keyStr + " " +
198         Bytes.toString(Bytes.toBytes(key)),
199           Arrays.equals(kv.getKey(), keyBytes));
200       byte [] valBytes = Bytes.toBytes(val);
201       assertTrue("bytes for vals do not match " + valStr + " " +
202         Bytes.toString(valBytes),
203         Arrays.equals(Bytes.toBytes(valStr), valBytes));
204       if (!scanner.next()) {
205         break;
206       }
207     }
208     assertEquals(i, start + n - 1);
209     return (start + n);
210   }
211 
212   private byte[] getSomeKey(int rowId) {
213     KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(),
214         Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
215     return kv.getKey();
216   }
217 
218   private void writeRecords(Writer writer, boolean useTags) throws IOException {
219     writeSomeRecords(writer, 0, 100, useTags);
220     writer.close();
221   }
222 
223   private FSDataOutputStream createFSOutput(Path name) throws IOException {
224     //if (fs.exists(name)) fs.delete(name, true);
225     FSDataOutputStream fout = fs.create(name);
226     return fout;
227   }
228 
229   /**
230    * test none codecs
231    * @param useTags 
232    */
233   void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
234     if (useTags) {
235       conf.setInt("hfile.format.version", 3);
236     }
237     if (cacheConf == null) cacheConf = new CacheConfig(conf);
238     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
239     FSDataOutputStream fout = createFSOutput(ncTFile);
240     HFileContext meta = new HFileContextBuilder()
241                         .withBlockSize(minBlockSize)
242                         .withCompression(AbstractHFileWriter.compressionByName(codec))
243                         .build();
244     Writer writer = HFile.getWriterFactory(conf, cacheConf)
245         .withOutputStream(fout)
246         .withFileContext(meta)
247         .withComparator(new KeyValue.KVComparator())
248         .create();
249     LOG.info(writer);
250     writeRecords(writer, useTags);
251     fout.close();
252     FSDataInputStream fin = fs.open(ncTFile);
253     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
254       fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
255     System.out.println(cacheConf.toString());
256     // Load up the index.
257     reader.loadFileInfo();
258     // Get a scanner that caches and that does not use pread.
259     HFileScanner scanner = reader.getScanner(true, false);
260     // Align scanner at start of the file.
261     scanner.seekTo();
262     readAllRecords(scanner);
263     int seekTo = scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50)));
264     System.out.println(seekTo);
265     assertTrue("location lookup failed",
266         scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))) == 0);
267     // read the key and see if it matches
268     ByteBuffer readKey = scanner.getKey();
269     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
270       Bytes.toBytes(readKey)));
271 
272     scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
273     ByteBuffer val1 = scanner.getValue();
274     scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
275     ByteBuffer val2 = scanner.getValue();
276     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
277 
278     reader.close();
279     fin.close();
280     fs.delete(ncTFile, true);
281   }
282 
283   public void testTFileFeatures() throws IOException {
284     testTFilefeaturesInternals(false);
285     testTFilefeaturesInternals(true);
286   }
287 
288   protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
289     basicWithSomeCodec("none", useTags);
290     basicWithSomeCodec("gz", useTags);
291   }
292 
293   private void writeNumMetablocks(Writer writer, int n) {
294     for (int i = 0; i < n; i++) {
295       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
296         private int val;
297         public Writable setVal(int val) { this.val = val; return this; }
298         
299         @Override
300         public void write(DataOutput out) throws IOException {
301           out.write(("something to test" + val).getBytes());
302         }
303         
304         @Override
305         public void readFields(DataInput in) throws IOException { }
306       }.setVal(i));
307     }
308   }
309 
310   private void someTestingWithMetaBlock(Writer writer) {
311     writeNumMetablocks(writer, 10);
312   }
313 
314   private void readNumMetablocks(Reader reader, int n) throws IOException {
315     for (int i = 0; i < n; i++) {
316       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
317       ByteBuffer expected = 
318         ByteBuffer.wrap(("something to test" + i).getBytes());
319       assertEquals("failed to match metadata",
320         Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
321     }
322   }
323 
324   private void someReadingWithMetaBlock(Reader reader) throws IOException {
325     readNumMetablocks(reader, 10);
326   }
327 
328   private void metablocks(final String compress) throws Exception {
329     if (cacheConf == null) cacheConf = new CacheConfig(conf);
330     Path mFile = new Path(ROOT_DIR, "meta.hfile");
331     FSDataOutputStream fout = createFSOutput(mFile);
332     HFileContext meta = new HFileContextBuilder()
333                         .withCompression(AbstractHFileWriter.compressionByName(compress))
334                         .withBlockSize(minBlockSize).build();
335     Writer writer = HFile.getWriterFactory(conf, cacheConf)
336         .withOutputStream(fout)
337         .withFileContext(meta)
338         .create();
339     someTestingWithMetaBlock(writer);
340     writer.close();
341     fout.close();
342     FSDataInputStream fin = fs.open(mFile);
343     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
344         this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
345     reader.loadFileInfo();
346     // No data -- this should return false.
347     assertFalse(reader.getScanner(false, false).seekTo());
348     someReadingWithMetaBlock(reader);
349     fs.delete(mFile, true);
350     reader.close();
351     fin.close();
352   }
353 
354   // test meta blocks for tfiles
355   public void testMetaBlocks() throws Exception {
356     metablocks("none");
357     metablocks("gz");
358   }
359 
360   public void testNullMetaBlocks() throws Exception {
361     if (cacheConf == null) cacheConf = new CacheConfig(conf);
362     for (Compression.Algorithm compressAlgo : 
363         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
364       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
365       FSDataOutputStream fout = createFSOutput(mFile);
366       HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
367                           .withBlockSize(minBlockSize).build();
368       Writer writer = HFile.getWriterFactory(conf, cacheConf)
369           .withOutputStream(fout)
370           .withFileContext(meta)
371           .create();
372       KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
373       writer.append(kv);
374       writer.close();
375       fout.close();
376       Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
377       reader.loadFileInfo();
378       assertNull(reader.getMetaBlock("non-existant", false));
379     }
380   }
381 
382   /**
383    * Make sure the ordinals for our compression algorithms do not change on us.
384    */
385   public void testCompressionOrdinance() {
386     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
387     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
388     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
389     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
390     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
391   }
392 
393 }
394