1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertTrue;
15
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.hadoop.hbase.client.Put;
21 import org.apache.hadoop.hbase.client.Result;
22 import org.apache.hadoop.hbase.client.ResultScanner;
23 import org.apache.hadoop.hbase.client.Scan;
24 import org.apache.hadoop.hbase.client.Table;
25 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
26 import org.apache.hadoop.hbase.client.metrics.ServerSideScanMetrics;
27 import org.apache.hadoop.hbase.filter.BinaryComparator;
28 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
29 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
30 import org.apache.hadoop.hbase.filter.Filter;
31 import org.apache.hadoop.hbase.filter.FilterList;
32 import org.apache.hadoop.hbase.filter.FilterList.Operator;
33 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
34 import org.apache.hadoop.hbase.filter.RowFilter;
35 import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
36 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
37 import org.apache.hadoop.hbase.testclassification.MediumTests;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44 @Category(MediumTests.class)
45 public class TestServerSideScanMetricsFromClientSide {
46 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47
48 private static Table TABLE = null;
49
50
51
52
53 private static TableName TABLE_NAME = TableName.valueOf("testTable");
54
55 private static int NUM_ROWS = 10;
56 private static byte[] ROW = Bytes.toBytes("testRow");
57 private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
58
59
60
61
62 private static int NUM_FAMILIES = 1;
63 private static byte[] FAMILY = Bytes.toBytes("testFamily");
64 private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
65
66 private static int NUM_QUALIFIERS = 1;
67 private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
68 private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
69
70 private static int VALUE_SIZE = 10;
71 private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
72
73 private static int NUM_COLS = NUM_FAMILIES * NUM_QUALIFIERS;
74
75
76
77 private static long CELL_HEAP_SIZE = -1;
78
79 @BeforeClass
80 public static void setUpBeforeClass() throws Exception {
81 TEST_UTIL.startMiniCluster(3);
82 TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
83 }
84
85 static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
86 byte[][] qualifiers, byte[] cellValue) throws IOException {
87 Table ht = TEST_UTIL.createTable(name, families);
88 List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
89 ht.put(puts);
90
91 return ht;
92 }
93
94 @AfterClass
95 public static void tearDownAfterClass() throws Exception {
96 TEST_UTIL.shutdownMiniCluster();
97 }
98
99
100
101
102
103
104
105
106
107
108 static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
109 byte[] value) throws IOException {
110 Put put;
111 ArrayList<Put> puts = new ArrayList<>();
112
113 for (int row = 0; row < rows.length; row++) {
114 put = new Put(rows[row]);
115 for (int fam = 0; fam < families.length; fam++) {
116 for (int qual = 0; qual < qualifiers.length; qual++) {
117 KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
118 put.add(kv);
119 }
120 }
121 puts.add(put);
122 }
123
124 return puts;
125 }
126
127
128
129
130
131
132
133 private long getCellHeapSize() throws Exception {
134 if (CELL_HEAP_SIZE == -1) {
135
136 Scan scan = new Scan();
137 scan.setMaxResultSize(1);
138 scan.setAllowPartialResults(true);
139 ResultScanner scanner = TABLE.getScanner(scan);
140
141 Result result = scanner.next();
142
143 assertTrue(result != null);
144 assertTrue(result.rawCells() != null);
145 assertTrue(result.rawCells().length == 1);
146
147 CELL_HEAP_SIZE = CellUtil.estimatedHeapSizeOf(result.rawCells()[0]);
148 scanner.close();
149 }
150
151 return CELL_HEAP_SIZE;
152 }
153
154 @Test
155 public void testRowsSeenMetric() throws Exception {
156
157 Scan baseScan;
158 baseScan = new Scan();
159 baseScan.setScanMetricsEnabled(true);
160 testRowsSeenMetric(baseScan);
161
162
163 baseScan.setCaching(1);
164 testRowsSeenMetric(baseScan);
165
166
167
168 baseScan.setMaxResultSize(1);
169 testRowsSeenMetric(baseScan);
170
171
172
173 baseScan.setCaching(NUM_ROWS);
174 baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
175 testRowsSeenMetric(baseScan);
176 }
177
178 public void testRowsSeenMetric(Scan baseScan) throws Exception {
179 Scan scan;
180 scan = new Scan(baseScan);
181 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, NUM_ROWS);
182
183 for (int i = 0; i < ROWS.length - 1; i++) {
184 scan = new Scan(baseScan);
185 scan.setStartRow(ROWS[0]);
186 scan.setStopRow(ROWS[i + 1]);
187 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, i + 1);
188 }
189
190 for (int i = ROWS.length - 1; i > 0; i--) {
191 scan = new Scan(baseScan);
192 scan.setStartRow(ROWS[i - 1]);
193 scan.setStopRow(ROWS[ROWS.length - 1]);
194 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length - i);
195 }
196
197
198 Filter filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator("xyz".getBytes()));
199 scan = new Scan(baseScan);
200 scan.setFilter(filter);
201 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
202
203
204 SingleColumnValueFilter singleColumnValueFilter =
205 new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.EQUAL, VALUE);
206 scan = new Scan(baseScan);
207 scan.setFilter(singleColumnValueFilter);
208 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
209
210
211 singleColumnValueFilter =
212 new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.NOT_EQUAL, VALUE);
213 scan = new Scan(baseScan);
214 scan.setFilter(singleColumnValueFilter);
215 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY, ROWS.length);
216 }
217
218 @Test
219 public void testRowsFilteredMetric() throws Exception {
220
221 Scan baseScan;
222 baseScan = new Scan();
223 baseScan.setScanMetricsEnabled(true);
224
225
226 testRowsFilteredMetric(baseScan);
227
228
229 baseScan.setCaching(1);
230 testRowsFilteredMetric(baseScan);
231
232
233
234 baseScan.setMaxResultSize(1);
235 testRowsFilteredMetric(baseScan);
236
237
238
239 baseScan.setCaching(NUM_ROWS);
240 baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1));
241 testRowsSeenMetric(baseScan);
242 }
243
244 public void testRowsFilteredMetric(Scan baseScan) throws Exception {
245 testRowsFilteredMetric(baseScan, null, 0);
246
247
248 Filter filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator("xyz".getBytes()));
249 testRowsFilteredMetric(baseScan, filter, ROWS.length);
250
251
252
253 filter = new FirstKeyOnlyFilter();
254 testRowsFilteredMetric(baseScan, filter, 0);
255
256
257
258 filter = new ColumnPrefixFilter(QUALIFIERS[0]);
259 testRowsFilteredMetric(baseScan, filter, 0);
260
261
262 filter = new ColumnPrefixFilter("xyz".getBytes());
263 testRowsFilteredMetric(baseScan, filter, ROWS.length);
264
265
266 filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.EQUAL, VALUE);
267 testRowsFilteredMetric(baseScan, filter, 0);
268
269
270 filter = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS[0], CompareOp.NOT_EQUAL, VALUE);
271 testRowsFilteredMetric(baseScan, filter, ROWS.length);
272
273 List<Filter> filters = new ArrayList<Filter>();
274 filters.add(new RowFilter(CompareOp.EQUAL, new BinaryComparator(ROWS[0])));
275 filters.add(new RowFilter(CompareOp.EQUAL, new BinaryComparator(ROWS[3])));
276 int numberOfMatchingRowFilters = filters.size();
277 filter = new FilterList(Operator.MUST_PASS_ONE, filters);
278 testRowsFilteredMetric(baseScan, filter, ROWS.length - numberOfMatchingRowFilters);
279 filters.clear();
280
281
282
283
284 for (int family = 0; family < FAMILIES.length; family++) {
285 for (int qualifier = 0; qualifier < QUALIFIERS.length; qualifier++) {
286 filters.add(new SingleColumnValueExcludeFilter(FAMILIES[family], QUALIFIERS[qualifier],
287 CompareOp.EQUAL, VALUE));
288 }
289 }
290 filter = new FilterList(Operator.MUST_PASS_ONE, filters);
291 testRowsFilteredMetric(baseScan, filter, ROWS.length);
292 }
293
294 public void testRowsFilteredMetric(Scan baseScan, Filter filter, int expectedNumFiltered)
295 throws Exception {
296 Scan scan = new Scan(baseScan);
297 if (filter != null) scan.setFilter(filter);
298 testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_FILTERED_KEY, expectedNumFiltered);
299 }
300
301
302
303
304
305
306
307
308 public void testMetric(Scan scan, String metricKey, long expectedValue) throws Exception {
309 assertTrue("Scan should be configured to record metrics", scan.isScanMetricsEnabled());
310 ResultScanner scanner = TABLE.getScanner(scan);
311
312
313 for (Result r : scanner) {
314 }
315 scanner.close();
316 ScanMetrics metrics = scan.getScanMetrics();
317 assertTrue("Metrics are null", metrics != null);
318 assertTrue("Metric : " + metricKey + " does not exist", metrics.hasCounter(metricKey));
319 final long actualMetricValue = metrics.getCounter(metricKey).get();
320 assertEquals("Metric: " + metricKey + " Expected: " + expectedValue + " Actual: "
321 + actualMetricValue, expectedValue, actualMetricValue);
322
323 }
324 }