View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import org.apache.commons.io.IOUtils;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.fs.FileUtil;
24  import org.apache.hadoop.fs.LocalFileSystem;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.testclassification.LargeTests;
28  import org.apache.hadoop.hbase.client.Put;
29  import org.apache.hadoop.hbase.client.Table;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32  import org.apache.hadoop.mapreduce.Job;
33  import org.apache.hadoop.util.GenericOptionsParser;
34  import org.junit.AfterClass;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  import java.io.*;
40  
41  import static org.junit.Assert.assertTrue;
42  import static org.junit.Assert.assertEquals;
43  import static org.junit.Assert.fail;
44  
45  @Category(LargeTests.class)
46  public class TestCellCounter {
47    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48    private static final byte[] ROW1 = Bytes.toBytes("row1");
49    private static final byte[] ROW2 = Bytes.toBytes("row2");
50    private static final String FAMILY_A_STRING = "a";
51    private static final String FAMILY_B_STRING = "b";
52    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54    private static final byte[] QUALIFIER = Bytes.toBytes("q");
55  
56    private static Path FQ_OUTPUT_DIR;
57    private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58        + "output";
59    private static long now = System.currentTimeMillis();
60  
61    @BeforeClass
62    public static void beforeClass() throws Exception {
63      UTIL.setJobWithoutMRCluster();
64      UTIL.startMiniCluster();
65      FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66      FileUtil.fullyDelete(new File(OUTPUT_DIR));
67    }
68  
69    @AfterClass
70    public static void afterClass() throws Exception {
71      UTIL.shutdownMiniCluster();
72    }
73  
74    /**
75     * Test CellCounter all data should print to output
76     *
77     */
78    @Test (timeout=300000)
79    public void testCellCounter() throws Exception {
80      String sourceTable = "sourceTable";
81      byte[][] families = { FAMILY_A, FAMILY_B };
82      Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
83      try{
84      Put p = new Put(ROW1);
85      p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
86      p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
87      p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
88      t.put(p);
89      p = new Put(ROW2);
90      p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
91      p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
92      p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
93      t.put(p);
94      String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
95      runCount(args);
96      FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
97          "part-r-00000");
98      String data = IOUtils.toString(inputStream);
99      inputStream.close();
100     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
101     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
102     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
103     assertTrue(data.contains("b;q" + "\t" + "1"));
104     assertTrue(data.contains("a;q" + "\t" + "1"));
105     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
106     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
107     }finally{
108       t.close();
109       FileUtil.fullyDelete(new File(OUTPUT_DIR));
110     }
111 
112   }
113 
114   /**
115    * Test CellCounter with time range all data should print to output
116    */
117   @Test (timeout=300000)
118   public void testCellCounterStartTimeRange() throws Exception {
119     String sourceTable = "testCellCounterStartTimeRange";
120     byte[][] families = { FAMILY_A, FAMILY_B };
121     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
122     try{
123     Put p = new Put(ROW1);
124     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
125     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
126     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
127     t.put(p);
128     p = new Put(ROW2);
129     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
130     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
131     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
132     t.put(p);
133     String[] args = {
134       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--starttime=" + now,
135       "--endtime=" + now + 2 };
136     runCount(args);
137     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
138         "part-r-00000");
139     String data = IOUtils.toString(inputStream);
140     inputStream.close();
141     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
142     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
143     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
144     assertTrue(data.contains("b;q" + "\t" + "1"));
145     assertTrue(data.contains("a;q" + "\t" + "1"));
146     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
147     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
148     }finally{
149       t.close();
150       FileUtil.fullyDelete(new File(OUTPUT_DIR));
151     }
152   }
153 
154   /**
155    * Test CellCounter with time range all data should print to output
156    */
157   @Test (timeout=300000)
158   public void testCellCounteEndTimeRange() throws Exception {
159     String sourceTable = "testCellCounterEndTimeRange";
160     byte[][] families = { FAMILY_A, FAMILY_B };
161     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
162     try{
163     Put p = new Put(ROW1);
164     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
165     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
166     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
167     t.put(p);
168     p = new Put(ROW2);
169     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
170     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
171     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
172     t.put(p);
173     String[] args = {
174       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--endtime=" + now + 1 };
175     runCount(args);
176     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
177         "part-r-00000");
178     String data = IOUtils.toString(inputStream);
179     inputStream.close();
180     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
181     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
182     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
183     assertTrue(data.contains("b;q" + "\t" + "1"));
184     assertTrue(data.contains("a;q" + "\t" + "1"));
185     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
186     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
187     }finally{
188       t.close();
189       FileUtil.fullyDelete(new File(OUTPUT_DIR));
190     }
191   }
192 
193    /**
194    * Test CellCounter with time range all data should print to output
195    */
196   @Test (timeout=300000)
197   public void testCellCounteOutOfTimeRange() throws Exception {
198     String sourceTable = "testCellCounterOutTimeRange";
199     byte[][] families = { FAMILY_A, FAMILY_B };
200     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
201     try{
202     Put p = new Put(ROW1);
203     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
204     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
205     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
206     t.put(p);
207     p = new Put(ROW2);
208     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
209     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
210     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
211     t.put(p);
212     String[] args = {
213       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "--starttime=" + now + 1,
214       "--endtime=" + now + 2 };
215 
216     runCount(args);
217     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
218         "part-r-00000");
219     String data = IOUtils.toString(inputStream);
220     inputStream.close();
221     // nothing should hace been emitted to the reducer
222     assertTrue(data.isEmpty());
223     }finally{
224       t.close();
225       FileUtil.fullyDelete(new File(OUTPUT_DIR));
226     }
227   }
228 
229 
230   private boolean runCount(String[] args) throws IOException, InterruptedException,
231       ClassNotFoundException {
232     // need to make a copy of the configuration because to make sure
233     // different temp dirs are used.
234     GenericOptionsParser opts = new GenericOptionsParser(
235         new Configuration(UTIL.getConfiguration()), args);
236     Configuration configuration = opts.getConfiguration();
237     args = opts.getRemainingArgs();
238     Job job = CellCounter.createSubmittableJob(configuration, args);
239     job.waitForCompletion(false);
240     return job.isSuccessful();
241   }
242 
243   /**
244    * Test main method of CellCounter
245    */
246   @Test (timeout=300000)
247   public void testCellCounterMain() throws Exception {
248 
249     PrintStream oldPrintStream = System.err;
250     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
251     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
252     System.setSecurityManager(newSecurityManager);
253     ByteArrayOutputStream data = new ByteArrayOutputStream();
254     String[] args = {};
255     System.setErr(new PrintStream(data));
256     try {
257       System.setErr(new PrintStream(data));
258 
259       try {
260         CellCounter.main(args);
261         fail("should be SecurityException");
262       } catch (SecurityException e) {
263         assertEquals(-1, newSecurityManager.getExitCode());
264         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
265         // should be information about usage
266         assertTrue(data.toString().contains("Usage:"));
267       }
268 
269     } finally {
270       System.setErr(oldPrintStream);
271       System.setSecurityManager(SECURITY_MANAGER);
272     }
273   }
274 
275   /**
276    * Test CellCounter for complete table all data should print to output
277    */
278   @Test(timeout = 300000)
279   public void testCellCounterForCompleteTable() throws Exception {
280     String sourceTable = "testCellCounterForCompleteTable";
281     String outputPath = OUTPUT_DIR + sourceTable;
282     LocalFileSystem localFileSystem = new LocalFileSystem();
283     Path outputDir =
284         new Path(outputPath).makeQualified(localFileSystem.getUri(),
285           localFileSystem.getWorkingDirectory());
286     byte[][] families = { FAMILY_A, FAMILY_B };
287     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
288     try {
289       Put p = new Put(ROW1);
290       p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
291       p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
292       p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
293       t.put(p);
294       p = new Put(ROW2);
295       p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
296       p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
297       p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
298       t.put(p);
299       String[] args = { sourceTable, outputDir.toString(), ";" };
300       runCount(args);
301       FileInputStream inputStream =
302           new FileInputStream(outputPath + File.separator + "part-r-00000");
303       String data = IOUtils.toString(inputStream);
304       inputStream.close();
305       assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
306       assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
307       assertTrue(data.contains("Total ROWS" + "\t" + "2"));
308       assertTrue(data.contains("b;q" + "\t" + "2"));
309       assertTrue(data.contains("a;q" + "\t" + "2"));
310       assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
311       assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
312       assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
313       assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
314     } finally {
315       t.close();
316       FileUtil.fullyDelete(new File(outputPath));
317     }
318   }
319 
320   @Test (timeout=300000)
321   public void TestCellCounterWithoutOutputDir() throws Exception {
322     PrintStream oldPrintStream = System.err;
323     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
324     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
325     System.setSecurityManager(newSecurityManager);
326     ByteArrayOutputStream data = new ByteArrayOutputStream();
327     String[] args = {"tableName"};
328     System.setErr(new PrintStream(data));
329     try {
330       System.setErr(new PrintStream(data));
331       try {
332         CellCounter.main(args);
333         fail("should be SecurityException");
334       } catch (SecurityException e) {
335         assertEquals(-1, newSecurityManager.getExitCode());
336         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
337         // should be information about usage
338         assertTrue(data.toString().contains("Usage:"));
339       }
340 
341     } finally {
342       System.setErr(oldPrintStream);
343       System.setSecurityManager(SECURITY_MANAGER);
344     }
345   }
346 }