1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import org.apache.commons.io.IOUtils;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FileUtil;
24 import org.apache.hadoop.fs.LocalFileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.testclassification.LargeTests;
28 import org.apache.hadoop.hbase.client.Put;
29 import org.apache.hadoop.hbase.client.Table;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32 import org.apache.hadoop.mapreduce.Job;
33 import org.apache.hadoop.util.GenericOptionsParser;
34 import org.junit.AfterClass;
35 import org.junit.BeforeClass;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39 import java.io.*;
40
41 import static org.junit.Assert.assertTrue;
42 import static org.junit.Assert.assertEquals;
43 import static org.junit.Assert.fail;
44
45 @Category(LargeTests.class)
46 public class TestCellCounter {
47 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48 private static final byte[] ROW1 = Bytes.toBytes("row1");
49 private static final byte[] ROW2 = Bytes.toBytes("row2");
50 private static final String FAMILY_A_STRING = "a";
51 private static final String FAMILY_B_STRING = "b";
52 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54 private static final byte[] QUALIFIER = Bytes.toBytes("q");
55
56 private static Path FQ_OUTPUT_DIR;
57 private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58 + "output";
59 private static long now = System.currentTimeMillis();
60
61 @BeforeClass
62 public static void beforeClass() throws Exception {
63 UTIL.setJobWithoutMRCluster();
64 UTIL.startMiniCluster();
65 FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66 FileUtil.fullyDelete(new File(OUTPUT_DIR));
67 }
68
69 @AfterClass
70 public static void afterClass() throws Exception {
71 UTIL.shutdownMiniCluster();
72 }
73
74
75
76
77
78 @Test (timeout=300000)
79 public void testCellCounter() throws Exception {
80 String sourceTable = "sourceTable";
81 byte[][] families = { FAMILY_A, FAMILY_B };
82 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
83 try{
84 Put p = new Put(ROW1);
85 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
86 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
87 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
88 t.put(p);
89 p = new Put(ROW2);
90 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
91 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
92 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
93 t.put(p);
94 String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
95 runCount(args);
96 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
97 "part-r-00000");
98 String data = IOUtils.toString(inputStream);
99 inputStream.close();
100 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
101 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
102 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
103 assertTrue(data.contains("b;q" + "\t" + "1"));
104 assertTrue(data.contains("a;q" + "\t" + "1"));
105 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
106 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
107 }finally{
108 t.close();
109 FileUtil.fullyDelete(new File(OUTPUT_DIR));
110 }
111
112 }
113
114
115
116
117 @Test (timeout=300000)
118 public void testCellCounterStartTimeRange() throws Exception {
119 String sourceTable = "testCellCounterStartTimeRange";
120 byte[][] families = { FAMILY_A, FAMILY_B };
121 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
122 try{
123 Put p = new Put(ROW1);
124 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
125 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
126 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
127 t.put(p);
128 p = new Put(ROW2);
129 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
130 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
131 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
132 t.put(p);
133 String[] args = {
134 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--starttime=" + now,
135 "--endtime=" + now + 2 };
136 runCount(args);
137 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
138 "part-r-00000");
139 String data = IOUtils.toString(inputStream);
140 inputStream.close();
141 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
142 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
143 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
144 assertTrue(data.contains("b;q" + "\t" + "1"));
145 assertTrue(data.contains("a;q" + "\t" + "1"));
146 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
147 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
148 }finally{
149 t.close();
150 FileUtil.fullyDelete(new File(OUTPUT_DIR));
151 }
152 }
153
154
155
156
157 @Test (timeout=300000)
158 public void testCellCounteEndTimeRange() throws Exception {
159 String sourceTable = "testCellCounterEndTimeRange";
160 byte[][] families = { FAMILY_A, FAMILY_B };
161 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
162 try{
163 Put p = new Put(ROW1);
164 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
165 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
166 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
167 t.put(p);
168 p = new Put(ROW2);
169 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
170 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
171 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
172 t.put(p);
173 String[] args = {
174 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1", "--endtime=" + now + 1 };
175 runCount(args);
176 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
177 "part-r-00000");
178 String data = IOUtils.toString(inputStream);
179 inputStream.close();
180 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
181 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
182 assertTrue(data.contains("Total ROWS" + "\t" + "1"));
183 assertTrue(data.contains("b;q" + "\t" + "1"));
184 assertTrue(data.contains("a;q" + "\t" + "1"));
185 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
186 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
187 }finally{
188 t.close();
189 FileUtil.fullyDelete(new File(OUTPUT_DIR));
190 }
191 }
192
193
194
195
196 @Test (timeout=300000)
197 public void testCellCounteOutOfTimeRange() throws Exception {
198 String sourceTable = "testCellCounterOutTimeRange";
199 byte[][] families = { FAMILY_A, FAMILY_B };
200 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
201 try{
202 Put p = new Put(ROW1);
203 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
204 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
205 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
206 t.put(p);
207 p = new Put(ROW2);
208 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
209 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
210 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
211 t.put(p);
212 String[] args = {
213 sourceTable, FQ_OUTPUT_DIR.toString(), ";", "--starttime=" + now + 1,
214 "--endtime=" + now + 2 };
215
216 runCount(args);
217 FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
218 "part-r-00000");
219 String data = IOUtils.toString(inputStream);
220 inputStream.close();
221
222 assertTrue(data.isEmpty());
223 }finally{
224 t.close();
225 FileUtil.fullyDelete(new File(OUTPUT_DIR));
226 }
227 }
228
229
230 private boolean runCount(String[] args) throws IOException, InterruptedException,
231 ClassNotFoundException {
232
233
234 GenericOptionsParser opts = new GenericOptionsParser(
235 new Configuration(UTIL.getConfiguration()), args);
236 Configuration configuration = opts.getConfiguration();
237 args = opts.getRemainingArgs();
238 Job job = CellCounter.createSubmittableJob(configuration, args);
239 job.waitForCompletion(false);
240 return job.isSuccessful();
241 }
242
243
244
245
246 @Test (timeout=300000)
247 public void testCellCounterMain() throws Exception {
248
249 PrintStream oldPrintStream = System.err;
250 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
251 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
252 System.setSecurityManager(newSecurityManager);
253 ByteArrayOutputStream data = new ByteArrayOutputStream();
254 String[] args = {};
255 System.setErr(new PrintStream(data));
256 try {
257 System.setErr(new PrintStream(data));
258
259 try {
260 CellCounter.main(args);
261 fail("should be SecurityException");
262 } catch (SecurityException e) {
263 assertEquals(-1, newSecurityManager.getExitCode());
264 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
265
266 assertTrue(data.toString().contains("Usage:"));
267 }
268
269 } finally {
270 System.setErr(oldPrintStream);
271 System.setSecurityManager(SECURITY_MANAGER);
272 }
273 }
274
275
276
277
278 @Test(timeout = 300000)
279 public void testCellCounterForCompleteTable() throws Exception {
280 String sourceTable = "testCellCounterForCompleteTable";
281 String outputPath = OUTPUT_DIR + sourceTable;
282 LocalFileSystem localFileSystem = new LocalFileSystem();
283 Path outputDir =
284 new Path(outputPath).makeQualified(localFileSystem.getUri(),
285 localFileSystem.getWorkingDirectory());
286 byte[][] families = { FAMILY_A, FAMILY_B };
287 Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
288 try {
289 Put p = new Put(ROW1);
290 p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
291 p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
292 p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
293 t.put(p);
294 p = new Put(ROW2);
295 p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
296 p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
297 p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
298 t.put(p);
299 String[] args = { sourceTable, outputDir.toString(), ";" };
300 runCount(args);
301 FileInputStream inputStream =
302 new FileInputStream(outputPath + File.separator + "part-r-00000");
303 String data = IOUtils.toString(inputStream);
304 inputStream.close();
305 assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
306 assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
307 assertTrue(data.contains("Total ROWS" + "\t" + "2"));
308 assertTrue(data.contains("b;q" + "\t" + "2"));
309 assertTrue(data.contains("a;q" + "\t" + "2"));
310 assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
311 assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
312 assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
313 assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
314 } finally {
315 t.close();
316 FileUtil.fullyDelete(new File(outputPath));
317 }
318 }
319
320 @Test (timeout=300000)
321 public void TestCellCounterWithoutOutputDir() throws Exception {
322 PrintStream oldPrintStream = System.err;
323 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
324 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
325 System.setSecurityManager(newSecurityManager);
326 ByteArrayOutputStream data = new ByteArrayOutputStream();
327 String[] args = {"tableName"};
328 System.setErr(new PrintStream(data));
329 try {
330 System.setErr(new PrintStream(data));
331 try {
332 CellCounter.main(args);
333 fail("should be SecurityException");
334 } catch (SecurityException e) {
335 assertEquals(-1, newSecurityManager.getExitCode());
336 assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
337
338 assertTrue(data.toString().contains("Usage:"));
339 }
340
341 } finally {
342 System.setErr(oldPrintStream);
343 System.setSecurityManager(SECURITY_MANAGER);
344 }
345 }
346 }