View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.Collection;
28  import java.util.Collections;
29  import java.util.Comparator;
30  import java.util.List;
31  import java.util.NavigableSet;
32  import java.util.TreeSet;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.Cell;
40  import org.apache.hadoop.hbase.CellUtil;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.KeyValue;
44  import org.apache.hadoop.hbase.KeyValueTestUtil;
45  import org.apache.hadoop.hbase.client.Put;
46  import org.apache.hadoop.hbase.client.Scan;
47  import org.apache.hadoop.hbase.io.compress.Compression;
48  import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
49  import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
50  import org.apache.hadoop.hbase.testclassification.SmallTests;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.junit.Before;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  import org.junit.runner.RunWith;
56  import org.junit.runners.Parameterized;
57  import org.junit.runners.Parameterized.Parameters;
58  
59  /**
60   * Test a multi-column scanner when there is a Bloom filter false-positive.
61   * This is needed for the multi-column Bloom filter optimization.
62   */
63  @RunWith(Parameterized.class)
64  @Category(SmallTests.class)
65  public class TestScanWithBloomError {
66  
67    private static final Log LOG =
68      LogFactory.getLog(TestScanWithBloomError.class);
69  
70    private static final String TABLE_NAME = "ScanWithBloomError";
71    private static final String FAMILY = "myCF";
72    private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
73    private static final String ROW = "theRow";
74    private static final String QUALIFIER_PREFIX = "qual";
75    private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
76    private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
77    private HRegion region;
78    private BloomType bloomType;
79    private FileSystem fs;
80    private Configuration conf;
81  
82    private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
83  
84    @Parameters
85    public static final Collection<Object[]> parameters() {
86      List<Object[]> configurations = new ArrayList<Object[]>();
87      for (BloomType bloomType : BloomType.values()) {
88        configurations.add(new Object[] { bloomType });
89      }
90      return configurations;
91    }
92  
93    public TestScanWithBloomError(BloomType bloomType) {
94      this.bloomType = bloomType;
95    }
96  
97    @Before
98    public void setUp() throws IOException{
99      conf = TEST_UTIL.getConfiguration();
100     fs = FileSystem.get(conf);
101   }
102 
103   @Test
104   public void testThreeStoreFiles() throws IOException {
105     region = TEST_UTIL.createTestRegion(TABLE_NAME,
106         new HColumnDescriptor(FAMILY)
107             .setCompressionType(Compression.Algorithm.GZ)
108             .setBloomFilterType(bloomType)
109             .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
110     createStoreFile(new int[] {1, 2, 6});
111     createStoreFile(new int[] {1, 2, 3, 7});
112     createStoreFile(new int[] {1, 9});
113     scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
114 
115     HRegion.closeHRegion(region);
116   }
117 
118   private void scanColSet(int[] colSet, int[] expectedResultCols)
119       throws IOException {
120     LOG.info("Scanning column set: " + Arrays.toString(colSet));
121     Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
122     addColumnSetToScan(scan, colSet);
123     RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
124     KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
125     assertEquals(0, storeHeap.getHeap().size());
126     StoreScanner storeScanner =
127         (StoreScanner) storeHeap.getCurrentForTesting();
128     @SuppressWarnings({ "unchecked", "rawtypes" })
129     List<StoreFileScanner> scanners = (List<StoreFileScanner>)
130         (List) storeScanner.getAllScannersForTesting();
131 
132     // Sort scanners by their HFile's modification time.
133     Collections.sort(scanners, new Comparator<StoreFileScanner>() {
134       @Override
135       public int compare(StoreFileScanner s1, StoreFileScanner s2) {
136         Path p1 = s1.getReader().getHFileReader().getPath();
137         Path p2 = s2.getReader().getHFileReader().getPath();
138         long t1, t2;
139         try {
140           t1 = fs.getFileStatus(p1).getModificationTime();
141           t2 = fs.getFileStatus(p2).getModificationTime();
142         } catch (IOException ex) {
143           throw new RuntimeException(ex);
144         }
145         return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
146       }
147     });
148 
149     StoreFile.Reader lastStoreFileReader = null;
150     for (StoreFileScanner sfScanner : scanners)
151       lastStoreFileReader = sfScanner.getReader();
152 
153     new HFilePrettyPrinter(conf).run(new String[]{ "-m", "-p", "-f",
154         lastStoreFileReader.getHFileReader().getPath().toString()});
155 
156     // Disable Bloom filter for the last store file. The disabled Bloom filter
157     // will always return "true".
158     LOG.info("Disabling Bloom filter for: "
159         + lastStoreFileReader.getHFileReader().getName());
160     lastStoreFileReader.disableBloomFilterForTesting();
161 
162     List<Cell> allResults = new ArrayList<Cell>();
163 
164     { // Limit the scope of results.
165       List<Cell> results = new ArrayList<Cell>();
166       while (scanner.next(results) || results.size() > 0) {
167         allResults.addAll(results);
168         results.clear();
169       }
170     }
171 
172     List<Integer> actualIds = new ArrayList<Integer>();
173     for (Cell kv : allResults) {
174       String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
175       assertTrue(qual.startsWith(QUALIFIER_PREFIX));
176       actualIds.add(Integer.valueOf(qual.substring(
177           QUALIFIER_PREFIX.length())));
178     }
179     List<Integer> expectedIds = new ArrayList<Integer>();
180     for (int expectedId : expectedResultCols)
181       expectedIds.add(expectedId);
182 
183     LOG.info("Column ids returned: " + actualIds + ", expected: "
184         + expectedIds);
185     assertEquals(expectedIds.toString(), actualIds.toString());
186   }
187 
188   private void addColumnSetToScan(Scan scan, int[] colIds) {
189     for (int colId : colIds) {
190       scan.addColumn(FAMILY_BYTES,
191           Bytes.toBytes(qualFromId(colId)));
192     }
193   }
194 
195   private String qualFromId(int colId) {
196     return QUALIFIER_PREFIX + colId;
197   }
198 
199   private void createStoreFile(int[] colIds)
200       throws IOException {
201     Put p = new Put(ROW_BYTES);
202     for (int colId : colIds) {
203       long ts = Long.MAX_VALUE;
204       String qual = qualFromId(colId);
205       allColIds.add(colId);
206       KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
207           qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
208       p.add(kv);
209     }
210     region.put(p);
211     region.flush(true);
212   }
213 
214 
215 }
216