View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.ResultScanner;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.client.Table;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.InternalScanner;
45  import org.apache.hadoop.hbase.regionserver.Region;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.FSTableDescriptors;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hdfs.MiniDFSCluster;
50  
51  /**
52   * Abstract HBase test class.  Initializes a few things that can come in handly
53   * like an HBaseConfiguration and filesystem.
54   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
55   */
56  public abstract class HBaseTestCase extends TestCase {
57    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
58  
59    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
60    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
61    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
62  
63    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
64  
65    private boolean localfs = false;
66    protected static Path testDir = null;
67    protected FileSystem fs = null;
68    protected HRegion meta = null;
69    protected static final char FIRST_CHAR = 'a';
70    protected static final char LAST_CHAR = 'z';
71    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
72    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
73    protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
74    protected static final int MAXVERSIONS = 3;
75  
76    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
77  
78    public volatile Configuration conf = HBaseConfiguration.create();
79    public final FSTableDescriptors fsTableDescriptors;
80    {
81      try {
82        fsTableDescriptors = new FSTableDescriptors(conf);
83      } catch (IOException e) {
84        throw new RuntimeException("Failed to init descriptors", e);
85      }
86    }
87  
88    /** constructor */
89    public HBaseTestCase() {
90      super();
91    }
92  
93    /**
94     * @param name
95     */
96    public HBaseTestCase(String name) {
97      super(name);
98    }
99  
100   /**
101    * Note that this method must be called after the mini hdfs cluster has
102    * started or we end up with a local file system.
103    */
104   @Override
105   protected void setUp() throws Exception {
106     super.setUp();
107     localfs =
108       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
109 
110     if (fs == null) {
111       this.fs = FileSystem.get(conf);
112     }
113     try {
114       if (localfs) {
115         this.testDir = getUnitTestdir(getName());
116         if (fs.exists(testDir)) {
117           fs.delete(testDir, true);
118         }
119       } else {
120         this.testDir = FSUtils.getRootDir(conf);
121       }
122     } catch (Exception e) {
123       LOG.fatal("error during setup", e);
124       throw e;
125     }
126   }
127 
128   @Override
129   protected void tearDown() throws Exception {
130     try {
131       if (localfs) {
132         if (this.fs.exists(testDir)) {
133           this.fs.delete(testDir, true);
134         }
135       }
136     } catch (Exception e) {
137       LOG.fatal("error during tear down", e);
138     }
139     super.tearDown();
140   }
141 
142   /**
143    * @see HBaseTestingUtility#getBaseTestDir
144    * @param testName
145    * @return directory to use for this test
146    */
147     protected Path getUnitTestdir(String testName) {
148       return testUtil.getDataTestDir(testName);
149     }
150 
151   /**
152    * You must call close on the returned region and then close on the log file
153    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getWAL()}
154    * and on it call close.
155    * @param desc
156    * @param startKey
157    * @param endKey
158    * @return An {@link HRegion}
159    * @throws IOException
160    */
161   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
162       byte [] endKey)
163   throws IOException {
164     return createNewHRegion(desc, startKey, endKey, this.conf);
165   }
166 
167   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
168       byte [] endKey, Configuration conf)
169   throws IOException {
170     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
171     return HRegion.createHRegion(hri, testDir, conf, desc);
172   }
173 
174   protected HRegion openClosedRegion(final HRegion closedRegion)
175   throws IOException {
176     return HRegion.openHRegion(closedRegion, null);
177   }
178 
179   /**
180    * Create a table of name <code>name</code> with {@link COLUMNS} for
181    * families.
182    * @param name Name to give table.
183    * @return Column descriptor.
184    */
185   protected HTableDescriptor createTableDescriptor(final String name) {
186     return createTableDescriptor(name, MAXVERSIONS);
187   }
188 
189   /**
190    * Create a table of name <code>name</code> with {@link COLUMNS} for
191    * families.
192    * @param name Name to give table.
193    * @param versions How many versions to allow per column.
194    * @return Column descriptor.
195    */
196   protected HTableDescriptor createTableDescriptor(final String name,
197       final int versions) {
198     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
199         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
200   }
201 
202   /**
203    * Create a table of name <code>name</code> with {@link COLUMNS} for
204    * families.
205    * @param name Name to give table.
206    * @param versions How many versions to allow per column.
207    * @return Column descriptor.
208    */
209   protected HTableDescriptor createTableDescriptor(final String name,
210       final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
211     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
212     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
213       htd.addFamily(new HColumnDescriptor(cfName)
214           .setMinVersions(minVersions)
215           .setMaxVersions(versions)
216           .setKeepDeletedCells(keepDeleted)
217           .setBlockCacheEnabled(false)
218           .setTimeToLive(ttl)
219       );
220     }
221     return htd;
222   }
223 
224   /**
225    * Add content to region <code>r</code> on the passed column
226    * <code>column</code>.
227    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
228    * @param r
229    * @param columnFamily
230    * @param column
231    * @throws IOException
232    * @return count of what we added.
233    */
234   public static long addContent(final Region r, final byte [] columnFamily, final byte[] column)
235   throws IOException {
236     byte [] startKey = r.getRegionInfo().getStartKey();
237     byte [] endKey = r.getRegionInfo().getEndKey();
238     byte [] startKeyBytes = startKey;
239     if (startKeyBytes == null || startKeyBytes.length == 0) {
240       startKeyBytes = START_KEY_BYTES;
241     }
242     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
243       startKeyBytes, endKey, -1);
244   }
245 
246   public static long addContent(final Region r, final byte [] columnFamily) throws IOException {
247     return addContent(r, columnFamily, null);
248   }
249 
250   /**
251    * Add content to region <code>r</code> on the passed column
252    * <code>column</code>.
253    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
254    * @param updater  An instance of {@link Incommon}.
255    * @param columnFamily
256    * @param writeToWAL
257    * @throws IOException
258    * @return count of what we added.
259    */
260   public static long addContent(final Incommon updater,
261       final String columnFamily) throws IOException {
262     return addContent(updater, columnFamily, START_KEY_BYTES, null);
263   }
264 
265   public static long addContent(final Incommon updater, final String family,
266       final String column) throws IOException {
267     return addContent(updater, family, column, START_KEY_BYTES, null);
268   }
269 
270   /**
271    * Add content to region <code>r</code> on the passed column
272    * <code>column</code>.
273    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
274    * @param updater  An instance of {@link Incommon}.
275    * @param columnFamily
276    * @param startKeyBytes Where to start the rows inserted
277    * @param endKey Where to stop inserting rows.
278    * @param writeToWAL
279    * @return count of what we added.
280    * @throws IOException
281    */
282   public static long addContent(final Incommon updater, final String columnFamily,
283       final byte [] startKeyBytes, final byte [] endKey)
284   throws IOException {
285     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
286   }
287 
288   public static long addContent(final Incommon updater, final String family, String column,
289       final byte [] startKeyBytes, final byte [] endKey) throws IOException {
290     return addContent(updater, family, column, startKeyBytes, endKey, -1);
291   }
292 
293   /**
294    * Add content to region <code>r</code> on the passed column
295    * <code>column</code>.
296    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
297    * @param updater  An instance of {@link Incommon}.
298    * @param column
299    * @param startKeyBytes Where to start the rows inserted
300    * @param endKey Where to stop inserting rows.
301    * @param ts Timestamp to write the content with.
302    * @param writeToWAL
303    * @return count of what we added.
304    * @throws IOException
305    */
306   public static long addContent(final Incommon updater,
307                                    final String columnFamily, 
308                                    final String column,
309       final byte [] startKeyBytes, final byte [] endKey, final long ts)
310   throws IOException {
311     long count = 0;
312     // Add rows of three characters.  The first character starts with the
313     // 'a' character and runs up to 'z'.  Per first character, we run the
314     // second character over same range.  And same for the third so rows
315     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
316     char secondCharStart = (char)startKeyBytes[1];
317     char thirdCharStart = (char)startKeyBytes[2];
318     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
319       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
320         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
321           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
322           if (endKey != null && endKey.length > 0
323               && Bytes.compareTo(endKey, t) <= 0) {
324             break EXIT;
325           }
326           try {
327             Put put;
328             if(ts != -1) {
329               put = new Put(t, ts);
330             } else {
331               put = new Put(t);
332             }
333             try {
334               StringBuilder sb = new StringBuilder();
335               if (column != null && column.contains(":")) {
336                 sb.append(column);
337               } else {
338                 if (columnFamily != null) {
339                   sb.append(columnFamily);
340                   if (!columnFamily.endsWith(":")) {
341                     sb.append(":");
342                   }
343                   if (column != null) {
344                     sb.append(column);
345                   }
346                 }
347               }
348               byte[][] split =
349                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
350               if(split.length == 1) {
351                 put.add(split[0], new byte[0], t);
352               } else {
353                 put.add(split[0], split[1], t);
354               }
355               put.setDurability(Durability.SKIP_WAL);
356               updater.put(put);
357               count++;
358             } catch (RuntimeException ex) {
359               ex.printStackTrace();
360               throw ex;
361             } catch (IOException ex) {
362               ex.printStackTrace();
363               throw ex;
364             }
365           } catch (RuntimeException ex) {
366             ex.printStackTrace();
367             throw ex;
368           } catch (IOException ex) {
369             ex.printStackTrace();
370             throw ex;
371           }
372         }
373         // Set start character back to FIRST_CHAR after we've done first loop.
374         thirdCharStart = FIRST_CHAR;
375       }
376       secondCharStart = FIRST_CHAR;
377     }
378     return count;
379   }
380 
381   /**
382    * Implementors can flushcache.
383    */
384   public interface FlushCache {
385     /**
386      * @throws IOException
387      */
388     void flushcache() throws IOException;
389   }
390 
391   /**
392    * Interface used by tests so can do common operations against an HTable
393    * or an HRegion.
394    *
395    * TOOD: Come up w/ a better name for this interface.
396    */
397   public interface Incommon {
398     /**
399      *
400      * @param delete
401      * @param writeToWAL
402      * @throws IOException
403      */
404     void delete(Delete delete, boolean writeToWAL)
405     throws IOException;
406 
407     /**
408      * @param put
409      * @throws IOException
410      */
411     void put(Put put) throws IOException;
412 
413     Result get(Get get) throws IOException;
414 
415     /**
416      * @param family
417      * @param qualifiers
418      * @param firstRow
419      * @param ts
420      * @return scanner for specified columns, first row and timestamp
421      * @throws IOException
422      */
423     ScannerIncommon getScanner(
424       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
425     )
426     throws IOException;
427   }
428 
429   /**
430    * A class that makes a {@link Incommon} out of a {@link HRegion}
431    */
432   public static class HRegionIncommon implements Incommon, FlushCache {
433     final HRegion region;
434 
435     /**
436      * @param HRegion
437      */
438     public HRegionIncommon(final HRegion HRegion) {
439       this.region = HRegion;
440     }
441 
442     public HRegionIncommon(final Region region) {
443       this.region = (HRegion)region;
444     }
445 
446     public void put(Put put) throws IOException {
447       region.put(put);
448     }
449 
450     public void delete(Delete delete,  boolean writeToWAL)
451     throws IOException {
452       this.region.delete(delete);
453     }
454 
455     public Result get(Get get) throws IOException {
456       return region.get(get);
457     }
458 
459     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
460         byte [] firstRow, long ts)
461       throws IOException {
462         Scan scan = new Scan(firstRow);
463         if(qualifiers == null || qualifiers.length == 0) {
464           scan.addFamily(family);
465         } else {
466           for(int i=0; i<qualifiers.length; i++){
467             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
468           }
469         }
470         scan.setTimeRange(0, ts);
471         return new
472           InternalScannerIncommon(region.getScanner(scan));
473       }
474 
475     public void flushcache() throws IOException {
476       this.region.flush(true);
477     }
478   }
479 
480   /**
481    * A class that makes a {@link Incommon} out of a {@link HTable}
482    */
483   public static class HTableIncommon implements Incommon {
484     final Table table;
485 
486     /**
487      * @param table
488      */
489     public HTableIncommon(final Table table) {
490       super();
491       this.table = table;
492     }
493 
494     public void put(Put put) throws IOException {
495       table.put(put);
496     }
497 
498 
499     public void delete(Delete delete, boolean writeToWAL)
500     throws IOException {
501       this.table.delete(delete);
502     }
503 
504     public Result get(Get get) throws IOException {
505       return table.get(get);
506     }
507 
508     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
509         byte [] firstRow, long ts)
510       throws IOException {
511       Scan scan = new Scan(firstRow);
512       if(qualifiers == null || qualifiers.length == 0) {
513         scan.addFamily(family);
514       } else {
515         for(int i=0; i<qualifiers.length; i++){
516           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
517         }
518       }
519       scan.setTimeRange(0, ts);
520       return new
521         ClientScannerIncommon(table.getScanner(scan));
522     }
523   }
524 
525   public interface ScannerIncommon
526   extends Iterable<Result> {
527     boolean next(List<Cell> values)
528     throws IOException;
529 
530     void close() throws IOException;
531   }
532 
533   public static class ClientScannerIncommon implements ScannerIncommon {
534     ResultScanner scanner;
535     public ClientScannerIncommon(ResultScanner scanner) {
536       this.scanner = scanner;
537     }
538 
539     @Override
540     public boolean next(List<Cell> values)
541     throws IOException {
542       Result results = scanner.next();
543       if (results == null) {
544         return false;
545       }
546       values.clear();
547       values.addAll(results.listCells());
548       return true;
549     }
550 
551     public void close() throws IOException {
552       scanner.close();
553     }
554 
555     public Iterator<Result> iterator() {
556       return scanner.iterator();
557     }
558   }
559 
560   public static class InternalScannerIncommon implements ScannerIncommon {
561     InternalScanner scanner;
562 
563     public InternalScannerIncommon(InternalScanner scanner) {
564       this.scanner = scanner;
565     }
566 
567     @Override
568     public boolean next(List<Cell> results)
569     throws IOException {
570       return scanner.next(results);
571     }
572 
573     @Override
574     public void close() throws IOException {
575       scanner.close();
576     }
577 
578     @Override
579     public Iterator<Result> iterator() {
580       throw new UnsupportedOperationException();
581     }
582   }
583 
584   protected void assertResultEquals(final HRegion region, final byte [] row,
585       final byte [] family, final byte [] qualifier, final long timestamp,
586       final byte [] value)
587     throws IOException {
588       Get get = new Get(row);
589       get.setTimeStamp(timestamp);
590       Result res = region.get(get);
591       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
592         res.getMap();
593       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
594 
595       if (value == null) {
596         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597             " at timestamp " + timestamp, null, res_value);
598       } else {
599         if (res_value == null) {
600           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601               " at timestamp " + timestamp + "\" was expected to be \"" +
602               Bytes.toStringBinary(value) + " but was null");
603         }
604         if (res_value != null) {
605           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
606               " at timestamp " +
607               timestamp, value, new String(res_value));
608         }
609       }
610     }
611 
612   /**
613    * Common method to close down a MiniDFSCluster and the associated file system
614    *
615    * @param cluster
616    */
617   public static void shutdownDfs(MiniDFSCluster cluster) {
618     if (cluster != null) {
619       LOG.info("Shutting down Mini DFS ");
620       try {
621         cluster.shutdown();
622       } catch (Exception e) {
623         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
624         // here because of an InterruptedException. Don't let exceptions in
625         // here be cause of test failure.
626       }
627       try {
628         FileSystem fs = cluster.getFileSystem();
629         if (fs != null) {
630           LOG.info("Shutting down FileSystem");
631           fs.close();
632         }
633         FileSystem.closeAll();
634       } catch (IOException e) {
635         LOG.error("error closing file system", e);
636       }
637     }
638   }
639 
640   /**
641    * You must call {@link #closeRootAndMeta()} when done after calling this
642    * method. It does cleanup.
643    * @throws IOException
644    */
645   protected void createMetaRegion() throws IOException {
646     FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
647     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
648       fsTableDescriptors.get(TableName.META_TABLE_NAME));
649   }
650 
651   protected void closeRootAndMeta() throws IOException {
652     HRegion.closeHRegion(meta);
653   }
654 
655   public static void assertByteEquals(byte[] expected,
656                                byte[] actual) {
657     if (Bytes.compareTo(expected, actual) != 0) {
658       throw new AssertionFailedError("expected:<" +
659       Bytes.toString(expected) + "> but was:<" +
660       Bytes.toString(actual) + ">");
661     }
662   }
663 
664   public static void assertEquals(byte[] expected,
665                                byte[] actual) {
666     if (Bytes.compareTo(expected, actual) != 0) {
667       throw new AssertionFailedError("expected:<" +
668       Bytes.toStringBinary(expected) + "> but was:<" +
669       Bytes.toStringBinary(actual) + ">");
670     }
671   }
672 
673 }