View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.List;
31  import java.util.Set;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.Cell;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.KeyValueTestUtil;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.client.Durability;
44  import org.apache.hadoop.hbase.client.Put;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.testclassification.SmallTests;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.junit.Rule;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  import org.junit.rules.TestName;
52  
53  @Category(SmallTests.class)
54  public class TestColumnSeeking {
55    @Rule public TestName name = new TestName();
56  
57    private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
58  
59    private static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
60  
61    @SuppressWarnings("unchecked")
62    @Test
63    public void testDuplicateVersions() throws IOException {
64      String family = "Family";
65      byte[] familyBytes = Bytes.toBytes("Family");
66      TableName table = TableName.valueOf(name.getMethodName());
67  
68      HColumnDescriptor hcd =
69          new HColumnDescriptor(familyBytes).setMaxVersions(1000);
70      hcd.setMaxVersions(3);
71      HTableDescriptor htd = new HTableDescriptor(table);
72      htd.addFamily(hcd);
73      HRegionInfo info = new HRegionInfo(table, null, null, false);
74      // Set this so that the archiver writes to the temp dir as well.
75      HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
76      try {
77        List<String> rows = generateRandomWords(10, "row");
78        List<String> allColumns = generateRandomWords(10, "column");
79        List<String> values = generateRandomWords(100, "value");
80  
81        long maxTimestamp = 2;
82        double selectPercent = 0.5;
83        int numberOfTests = 5;
84        double flushPercentage = 0.2;
85        double minorPercentage = 0.2;
86        double majorPercentage = 0.2;
87        double putPercentage = 0.2;
88  
89        HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
90  
91        HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
92        ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
93  
94        for (int i = 0; i < numberOfTests; i++) {
95          kvMaps[i] = new HashMap<String, KeyValue>();
96          columnLists[i] = new ArrayList<String>();
97          for (String column : allColumns) {
98            if (Math.random() < selectPercent) {
99              columnLists[i].add(column);
100           }
101         }
102       }
103 
104       for (String value : values) {
105         for (String row : rows) {
106           Put p = new Put(Bytes.toBytes(row));
107           p.setDurability(Durability.SKIP_WAL);
108           for (String column : allColumns) {
109             for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
110               KeyValue kv =
111                   KeyValueTestUtil.create(row, family, column, timestamp, value);
112               if (Math.random() < putPercentage) {
113                 p.add(kv);
114                 allKVMap.put(kv.getKeyString(), kv);
115                 for (int i = 0; i < numberOfTests; i++) {
116                   if (columnLists[i].contains(column)) {
117                     kvMaps[i].put(kv.getKeyString(), kv);
118                   }
119                 }
120               }
121             }
122           }
123           region.put(p);
124           if (Math.random() < flushPercentage) {
125             LOG.info("Flushing... ");
126             region.flush(true);
127           }
128 
129           if (Math.random() < minorPercentage) {
130             LOG.info("Minor compacting... ");
131             region.compact(false);
132           }
133 
134           if (Math.random() < majorPercentage) {
135             LOG.info("Major compacting... ");
136             region.compact(true);
137           }
138         }
139       }
140 
141       for (int i = 0; i < numberOfTests + 1; i++) {
142         Collection<KeyValue> kvSet;
143         Scan scan = new Scan();
144         scan.setMaxVersions();
145         if (i < numberOfTests) {
146           if (columnLists[i].size() == 0) continue; // HBASE-7700
147           kvSet = kvMaps[i].values();
148           for (String column : columnLists[i]) {
149             scan.addColumn(familyBytes, Bytes.toBytes(column));
150           }
151           LOG.info("ExplicitColumns scanner");
152           LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
153               + kvSet.size());
154         } else {
155           kvSet = allKVMap.values();
156           LOG.info("Wildcard scanner");
157           LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
158 
159         }
160         InternalScanner scanner = region.getScanner(scan);
161         List<Cell> results = new ArrayList<Cell>();
162         while (scanner.next(results))
163           ;
164         assertEquals(kvSet.size(), results.size());
165         assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
166       }
167     } finally {
168       HRegion.closeHRegion(region);
169     }
170 
171     HRegion.closeHRegion(region);
172   }
173 
174   @SuppressWarnings("unchecked")
175   @Test
176   public void testReseeking() throws IOException {
177     String family = "Family";
178     byte[] familyBytes = Bytes.toBytes("Family");
179     TableName table = TableName.valueOf(name.getMethodName());
180 
181     HTableDescriptor htd = new HTableDescriptor(table);
182     HColumnDescriptor hcd = new HColumnDescriptor(family);
183     hcd.setMaxVersions(3);
184     htd.addFamily(hcd);
185 
186     HRegionInfo info = new HRegionInfo(table, null, null, false);
187     HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
188 
189     List<String> rows = generateRandomWords(10, "row");
190     List<String> allColumns = generateRandomWords(100, "column");
191 
192     long maxTimestamp = 2;
193     double selectPercent = 0.5;
194     int numberOfTests = 5;
195     double flushPercentage = 0.2;
196     double minorPercentage = 0.2;
197     double majorPercentage = 0.2;
198     double putPercentage = 0.2;
199 
200     HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
201 
202     HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
203     ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
204     String valueString = "Value";
205 
206     for (int i = 0; i < numberOfTests; i++) {
207       kvMaps[i] = new HashMap<String, KeyValue>();
208       columnLists[i] = new ArrayList<String>();
209       for (String column : allColumns) {
210         if (Math.random() < selectPercent) {
211           columnLists[i].add(column);
212         }
213       }
214     }
215 
216     for (String row : rows) {
217       Put p = new Put(Bytes.toBytes(row));
218       p.setDurability(Durability.SKIP_WAL);
219       for (String column : allColumns) {
220         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
221           KeyValue kv =
222               KeyValueTestUtil.create(row, family, column, timestamp,
223                   valueString);
224           if (Math.random() < putPercentage) {
225             p.add(kv);
226             allKVMap.put(kv.getKeyString(), kv);
227             for (int i = 0; i < numberOfTests; i++) {
228               if (columnLists[i].contains(column)) {
229                 kvMaps[i].put(kv.getKeyString(), kv);
230               }
231             }
232           }
233 
234         }
235       }
236       region.put(p);
237       if (Math.random() < flushPercentage) {
238         LOG.info("Flushing... ");
239         region.flush(true);
240       }
241 
242       if (Math.random() < minorPercentage) {
243         LOG.info("Minor compacting... ");
244         region.compact(false);
245       }
246 
247       if (Math.random() < majorPercentage) {
248         LOG.info("Major compacting... ");
249         region.compact(true);
250       }
251     }
252 
253     for (int i = 0; i < numberOfTests + 1; i++) {
254       Collection<KeyValue> kvSet;
255       Scan scan = new Scan();
256       scan.setMaxVersions();
257       if (i < numberOfTests) {
258         if (columnLists[i].size() == 0) continue; // HBASE-7700
259         kvSet = kvMaps[i].values();
260         for (String column : columnLists[i]) {
261           scan.addColumn(familyBytes, Bytes.toBytes(column));
262         }
263         LOG.info("ExplicitColumns scanner");
264         LOG.info("Columns: " + columnLists[i].size() + "  Keys: "
265             + kvSet.size());
266       } else {
267         kvSet = allKVMap.values();
268         LOG.info("Wildcard scanner");
269         LOG.info("Columns: " + allColumns.size() + "  Keys: " + kvSet.size());
270 
271       }
272       InternalScanner scanner = region.getScanner(scan);
273       List<Cell> results = new ArrayList<Cell>();
274       while (scanner.next(results))
275         ;
276       assertEquals(kvSet.size(), results.size());
277       assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
278     }
279 
280     HRegion.closeHRegion(region);
281   }
282 
283   List<String> generateRandomWords(int numberOfWords, String suffix) {
284     Set<String> wordSet = new HashSet<String>();
285     for (int i = 0; i < numberOfWords; i++) {
286       int lengthOfWords = (int) (Math.random() * 5) + 1;
287       char[] wordChar = new char[lengthOfWords];
288       for (int j = 0; j < wordChar.length; j++) {
289         wordChar[j] = (char) (Math.random() * 26 + 97);
290       }
291       String word;
292       if (suffix == null) {
293         word = new String(wordChar);
294       } else {
295         word = new String(wordChar) + suffix;
296       }
297       wordSet.add(word);
298     }
299     List<String> wordList = new ArrayList<String>(wordSet);
300     return wordList;
301   }
302 
303 }
304