View Javadoc

1   /*
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.filter;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.commons.codec.binary.Hex;
25  import org.apache.hadoop.hbase.client.Put;
26  import org.apache.hadoop.hbase.client.Result;
27  import org.apache.hadoop.hbase.client.ResultScanner;
28  import org.apache.hadoop.hbase.client.Scan;
29  import org.apache.hadoop.hbase.client.Table;
30  import org.apache.hadoop.hbase.testclassification.MediumTests;
31  import org.junit.Assert;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  
35  import java.io.IOException;
36  import java.util.ArrayList;
37  import java.util.Arrays;
38  import java.util.List;
39  
40  /**
41   * Test if Scan.setRowPrefixFilter works as intended.
42   */
43  @Category({MediumTests.class})
44  public class TestScanRowPrefix extends FilterTestingCluster {
45    private static final Log LOG = LogFactory
46        .getLog(TestScanRowPrefix.class);
47  
48    @Test
49    public void testPrefixScanning() throws IOException {
50      String tableName = "prefixScanning";
51      createTable(tableName,"F");
52      Table table = openTable(tableName);
53  
54      /**
55       * Note that about half of these tests were relevant for an different implementation approach
56       * of setRowPrefixFilter. These test cases have been retained to ensure that also the
57       * edge cases found there are still covered.
58       */
59  
60      final byte[][] rowIds = {
61          {(byte) 0x11},                                                      //  0
62          {(byte) 0x12},                                                      //  1
63          {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFE},               //  2
64          {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF},               //  3
65          {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x00},  //  4
66          {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF, (byte) 0x01},  //  5
67          {(byte) 0x12, (byte) 0x24},                                         //  6
68          {(byte) 0x12, (byte) 0x24, (byte) 0x00},                            //  7
69          {(byte) 0x12, (byte) 0x24, (byte) 0x00, (byte) 0x00},               //  8
70          {(byte) 0x12, (byte) 0x25},                                         //  9
71          {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF},  // 10
72      };
73      for (byte[] rowId: rowIds) {
74        Put p = new Put(rowId);
75        // Use the rowId as the column qualifier
76        p.add("F".getBytes(), rowId, "Dummy value".getBytes());
77        table.put(p);
78      }
79  
80      byte[] prefix0 = {};
81      List<byte[]> expected0 = new ArrayList<>(16);
82      expected0.addAll(Arrays.asList(rowIds)); // Expect all rows
83  
84      byte[] prefix1 = {(byte) 0x12, (byte) 0x23};
85      List<byte[]> expected1 = new ArrayList<>(16);
86      expected1.add(rowIds[2]);
87      expected1.add(rowIds[3]);
88      expected1.add(rowIds[4]);
89      expected1.add(rowIds[5]);
90  
91      byte[] prefix2 = {(byte) 0x12, (byte) 0x23, (byte) 0xFF, (byte) 0xFF};
92      List<byte[]> expected2 = new ArrayList<>();
93      expected2.add(rowIds[3]);
94      expected2.add(rowIds[4]);
95      expected2.add(rowIds[5]);
96  
97      byte[] prefix3 = {(byte) 0x12, (byte) 0x24};
98      List<byte[]> expected3 = new ArrayList<>();
99      expected3.add(rowIds[6]);
100     expected3.add(rowIds[7]);
101     expected3.add(rowIds[8]);
102 
103     byte[] prefix4 = {(byte) 0xFF, (byte) 0xFF};
104     List<byte[]> expected4 = new ArrayList<>();
105     expected4.add(rowIds[10]);
106 
107     // ========
108     // PREFIX 0
109     Scan scan = new Scan();
110     scan.setRowPrefixFilter(prefix0);
111     verifyScanResult(table, scan, expected0, "Scan empty prefix failed");
112 
113     // ========
114     // PREFIX 1
115     scan = new Scan();
116     scan.setRowPrefixFilter(prefix1);
117     verifyScanResult(table, scan, expected1, "Scan normal prefix failed");
118 
119     scan.setRowPrefixFilter(null);
120     verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
121 
122     scan = new Scan();
123     scan.setFilter(new ColumnPrefixFilter(prefix1));
124     verifyScanResult(table, scan, expected1, "Double check on column prefix failed");
125 
126     // ========
127     // PREFIX 2
128     scan = new Scan();
129     scan.setRowPrefixFilter(prefix2);
130     verifyScanResult(table, scan, expected2, "Scan edge 0xFF prefix failed");
131 
132     scan.setRowPrefixFilter(null);
133     verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
134 
135     scan = new Scan();
136     scan.setFilter(new ColumnPrefixFilter(prefix2));
137     verifyScanResult(table, scan, expected2, "Double check on column prefix failed");
138 
139     // ========
140     // PREFIX 3
141     scan = new Scan();
142     scan.setRowPrefixFilter(prefix3);
143     verifyScanResult(table, scan, expected3, "Scan normal with 0x00 ends failed");
144 
145     scan.setRowPrefixFilter(null);
146     verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
147 
148     scan = new Scan();
149     scan.setFilter(new ColumnPrefixFilter(prefix3));
150     verifyScanResult(table, scan, expected3, "Double check on column prefix failed");
151 
152     // ========
153     // PREFIX 4
154     scan = new Scan();
155     scan.setRowPrefixFilter(prefix4);
156     verifyScanResult(table, scan, expected4, "Scan end prefix failed");
157 
158     scan.setRowPrefixFilter(null);
159     verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
160 
161     scan = new Scan();
162     scan.setFilter(new ColumnPrefixFilter(prefix4));
163     verifyScanResult(table, scan, expected4, "Double check on column prefix failed");
164 
165     // ========
166     // COMBINED
167     // Prefix + Filter
168     scan = new Scan();
169     scan.setRowPrefixFilter(prefix1);
170     verifyScanResult(table, scan, expected1, "Prefix filter failed");
171 
172     scan.setFilter(new ColumnPrefixFilter(prefix2));
173     verifyScanResult(table, scan, expected2, "Combined Prefix + Filter failed");
174 
175     scan.setRowPrefixFilter(null);
176     verifyScanResult(table, scan, expected2, "Combined Prefix + Filter; removing Prefix failed");
177 
178     scan.setFilter(null);
179     verifyScanResult(table, scan, expected0, "Scan after Filter reset failed");
180 
181     // ========
182     // Reversed: Filter + Prefix
183     scan = new Scan();
184     scan.setFilter(new ColumnPrefixFilter(prefix2));
185     verifyScanResult(table, scan, expected2, "Test filter failed");
186 
187     scan.setRowPrefixFilter(prefix1);
188     verifyScanResult(table, scan, expected2, "Combined Filter + Prefix failed");
189 
190     scan.setFilter(null);
191     verifyScanResult(table, scan, expected1, "Combined Filter + Prefix ; removing Filter failed");
192 
193     scan.setRowPrefixFilter(null);
194     verifyScanResult(table, scan, expected0, "Scan after prefix reset failed");
195   }
196 
197   private void verifyScanResult(Table table, Scan scan, List<byte[]> expectedKeys, String message) {
198     List<byte[]> actualKeys = new ArrayList<>();
199     try {
200       ResultScanner scanner = table.getScanner(scan);
201       for (Result result : scanner) {
202         actualKeys.add(result.getRow());
203       }
204 
205       String fullMessage = message;
206       if (LOG.isDebugEnabled()) {
207         fullMessage = message + "\n" + tableOfTwoListsOfByteArrays(
208                 "Expected", expectedKeys,
209                 "Actual  ", actualKeys);
210       }
211 
212       Assert.assertArrayEquals(
213               fullMessage,
214               expectedKeys.toArray(),
215               actualKeys.toArray());
216     } catch (IOException e) {
217       e.printStackTrace();
218       Assert.fail();
219     }
220   }
221 
222   private String printMultiple(char letter, int count) {
223     StringBuilder sb = new StringBuilder(count);
224     for (int i = 0; i < count; i++) {
225       sb.append(letter);
226     }
227     return sb.toString();
228   }
229 
230   private String tableOfTwoListsOfByteArrays(
231           String label1, List<byte[]> listOfBytes1,
232           String label2, List<byte[]> listOfBytes2) {
233     int margin1 = calculateWidth(label1, listOfBytes1);
234     int margin2 = calculateWidth(label2, listOfBytes2);
235 
236     StringBuilder sb = new StringBuilder(512);
237     String separator = '+' + printMultiple('-', margin1 + margin2 + 5) + '+' + '\n';
238     sb.append(separator);
239     sb.append(printLine(label1, margin1, label2, margin2)).append('\n');
240     sb.append(separator);
241     int maxLength = Math.max(listOfBytes1.size(), listOfBytes2.size());
242     for (int offset = 0; offset < maxLength; offset++) {
243       String value1 = getStringFromList(listOfBytes1, offset);
244       String value2 = getStringFromList(listOfBytes2, offset);
245       sb.append(printLine(value1, margin1, value2, margin2)).append('\n');
246     }
247     sb.append(separator).append('\n');
248     return sb.toString();
249   }
250 
251   private String printLine(String leftValue, int leftWidth1, String rightValue, int rightWidth) {
252     return "| " +
253            leftValue  + printMultiple(' ', leftWidth1 - leftValue.length() ) +
254            " | " +
255            rightValue + printMultiple(' ', rightWidth - rightValue.length()) +
256            " |";
257   }
258 
259   private int calculateWidth(String label1, List<byte[]> listOfBytes1) {
260     int longestList1 = label1.length();
261     for (byte[] value : listOfBytes1) {
262       longestList1 = Math.max(value.length * 2, longestList1);
263     }
264     return longestList1 + 5;
265   }
266 
267   private String getStringFromList(List<byte[]> listOfBytes, int offset) {
268     String value1;
269     if (listOfBytes.size() > offset) {
270       value1 = Hex.encodeHexString(listOfBytes.get(offset));
271     } else {
272       value1 = "<missing>";
273     }
274     return value1;
275   }
276 
277 }