View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.thrift2;
19  
20  import static java.nio.ByteBuffer.wrap;
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.nio.ByteBuffer;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.ArrayList;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.List;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.testclassification.MediumTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.client.Admin;
43  import org.apache.hadoop.hbase.client.Connection;
44  import org.apache.hadoop.hbase.client.ConnectionFactory;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
47  import org.apache.hadoop.hbase.security.User;
48  import org.apache.hadoop.hbase.security.UserProvider;
49  import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
50  import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
51  import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
52  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
53  import org.apache.hadoop.hbase.security.visibility.VisibilityController;
54  import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
55  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
56  import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
57  import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
58  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
59  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
60  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
61  import org.apache.hadoop.hbase.thrift2.generated.TGet;
62  import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
63  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
64  import org.apache.hadoop.hbase.thrift2.generated.TPut;
65  import org.apache.hadoop.hbase.thrift2.generated.TResult;
66  import org.apache.hadoop.hbase.thrift2.generated.TScan;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.junit.AfterClass;
69  import org.junit.Assert;
70  import org.junit.Before;
71  import org.junit.BeforeClass;
72  import org.junit.Test;
73  import org.junit.experimental.categories.Category;
74  
75  @Category(MediumTests.class)
76  public class TestThriftHBaseServiceHandlerWithLabels {
77  
78    private static final Log LOG = LogFactory
79      .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
80  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
81  
82  // Static names for tables, columns, rows, and values
83  private static byte[] tableAname = Bytes.toBytes("tableA");
84  private static byte[] familyAname = Bytes.toBytes("familyA");
85  private static byte[] familyBname = Bytes.toBytes("familyB");
86  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
87  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
88  private static byte[] valueAname = Bytes.toBytes("valueA");
89  private static byte[] valueBname = Bytes.toBytes("valueB");
90  private static HColumnDescriptor[] families = new HColumnDescriptor[] {
91      new HColumnDescriptor(familyAname).setMaxVersions(3),
92      new HColumnDescriptor(familyBname).setMaxVersions(2) };
93  
94  private final static String TOPSECRET = "topsecret";
95  private final static String PUBLIC = "public";
96  private final static String PRIVATE = "private";
97  private final static String CONFIDENTIAL = "confidential";
98  private final static String SECRET = "secret";
99  private static User SUPERUSER;
100 
101 private static Configuration conf;
102 
103 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
104     List<TColumnValue> columnValuesB) {
105   assertEquals(columnValuesA.size(), columnValuesB.size());
106   Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
107     @Override
108     public int compare(TColumnValue o1, TColumnValue o2) {
109       return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
110           Bytes.add(o2.getFamily(), o2.getQualifier()));
111     }
112   };
113   Collections.sort(columnValuesA, comparator);
114   Collections.sort(columnValuesB, comparator);
115 
116   for (int i = 0; i < columnValuesA.size(); i++) {
117     TColumnValue a = columnValuesA.get(i);
118     TColumnValue b = columnValuesB.get(i);
119     assertArrayEquals(a.getFamily(), b.getFamily());
120     assertArrayEquals(a.getQualifier(), b.getQualifier());
121     assertArrayEquals(a.getValue(), b.getValue());
122   }
123 }
124 
125 @BeforeClass
126 public static void beforeClass() throws Exception {
127   SUPERUSER = User.createUserForTesting(conf, "admin",
128       new String[] { "supergroup" });
129   conf = UTIL.getConfiguration();
130   conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
131       SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
132   conf.set("hbase.superuser", SUPERUSER.getShortName());
133   conf.set("hbase.coprocessor.master.classes",
134       VisibilityController.class.getName());
135   conf.set("hbase.coprocessor.region.classes",
136       VisibilityController.class.getName());
137   conf.setInt("hfile.format.version", 3);
138   UTIL.startMiniCluster(1);
139   // Wait for the labels table to become available
140   UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
141   createLabels();
142   Admin admin = new HBaseAdmin(UTIL.getConfiguration());
143   HTableDescriptor tableDescriptor = new HTableDescriptor(
144       TableName.valueOf(tableAname));
145   for (HColumnDescriptor family : families) {
146     tableDescriptor.addFamily(family);
147   }
148   admin.createTable(tableDescriptor);
149   admin.close();
150   setAuths();
151 }
152 
153 private static void createLabels() throws IOException, InterruptedException {
154   PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
155     public VisibilityLabelsResponse run() throws Exception {
156       String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
157       try (Connection conn = ConnectionFactory.createConnection(conf)) {
158         VisibilityClient.addLabels(conn, labels);
159       } catch (Throwable t) {
160         throw new IOException(t);
161       }
162       return null;
163     }
164   };
165   SUPERUSER.runAs(action);
166 }
167 
168 private static void setAuths() throws IOException {
169   String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
170   try {
171     VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
172   } catch (Throwable t) {
173     throw new IOException(t);
174   }
175 }
176 
177 @AfterClass
178 public static void afterClass() throws Exception {
179   UTIL.shutdownMiniCluster();
180 }
181 
182 @Before
183 public void setup() throws Exception {
184 
185 }
186 
187 private ThriftHBaseServiceHandler createHandler() throws IOException {
188   return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
189 }
190 
191 @Test
192 public void testScanWithVisibilityLabels() throws Exception {
193   ThriftHBaseServiceHandler handler = createHandler();
194   ByteBuffer table = wrap(tableAname);
195 
196   // insert data
197   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
198       wrap(qualifierAname), wrap(valueAname));
199   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
200   columnValues.add(columnValue);
201   for (int i = 0; i < 10; i++) {
202     TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
203     if (i == 5) {
204       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
205     } else {
206       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
207           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
208     }
209     handler.put(table, put);
210   }
211 
212   // create scan instance
213   TScan scan = new TScan();
214   List<TColumn> columns = new ArrayList<TColumn>();
215   TColumn column = new TColumn();
216   column.setFamily(familyAname);
217   column.setQualifier(qualifierAname);
218   columns.add(column);
219   scan.setColumns(columns);
220   scan.setStartRow("testScan".getBytes());
221   scan.setStopRow("testScan\uffff".getBytes());
222 
223   TAuthorization tauth = new TAuthorization();
224   List<String> labels = new ArrayList<String>();
225   labels.add(SECRET);
226   labels.add(PRIVATE);
227   tauth.setLabels(labels);
228   scan.setAuthorizations(tauth);
229   // get scanner and rows
230   int scanId = handler.openScanner(table, scan);
231   List<TResult> results = handler.getScannerRows(scanId, 10);
232   assertEquals(9, results.size());
233   Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
234       ("testScan" + 5).getBytes()));
235   for (int i = 0; i < 9; i++) {
236     if (i < 5) {
237       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
238     } else if (i == 5) {
239       continue;
240     } else {
241       assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
242           .getRow());
243     }
244   }
245 
246   // check that we are at the end of the scan
247   results = handler.getScannerRows(scanId, 9);
248   assertEquals(0, results.size());
249 
250   // close scanner and check that it was indeed closed
251   handler.closeScanner(scanId);
252   try {
253     handler.getScannerRows(scanId, 9);
254     fail("Scanner id should be invalid");
255   } catch (TIllegalArgument e) {
256   }
257 }
258 
259 @Test
260 public void testGetScannerResultsWithAuthorizations() throws Exception {
261   ThriftHBaseServiceHandler handler = createHandler();
262   ByteBuffer table = wrap(tableAname);
263 
264   // insert data
265   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
266       wrap(qualifierAname), wrap(valueAname));
267   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
268   columnValues.add(columnValue);
269   for (int i = 0; i < 20; i++) {
270     TPut put = new TPut(
271         wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
272         columnValues);
273     if (i == 3) {
274       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
275     } else {
276       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
277           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
278     }
279     handler.put(table, put);
280   }
281 
282   // create scan instance
283   TScan scan = new TScan();
284   List<TColumn> columns = new ArrayList<TColumn>();
285   TColumn column = new TColumn();
286   column.setFamily(familyAname);
287   column.setQualifier(qualifierAname);
288   columns.add(column);
289   scan.setColumns(columns);
290   scan.setStartRow("testGetScannerResults".getBytes());
291 
292   // get 5 rows and check the returned results
293   scan.setStopRow("testGetScannerResults05".getBytes());
294   TAuthorization tauth = new TAuthorization();
295   List<String> labels = new ArrayList<String>();
296   labels.add(SECRET);
297   labels.add(PRIVATE);
298   tauth.setLabels(labels);
299   scan.setAuthorizations(tauth);
300   List<TResult> results = handler.getScannerResults(table, scan, 5);
301   assertEquals(4, results.size());
302   for (int i = 0; i < 4; i++) {
303     if (i < 3) {
304       assertArrayEquals(
305           ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
306           results.get(i).getRow());
307     } else if (i == 3) {
308       continue;
309     } else {
310       assertArrayEquals(
311           ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
312               .get(i).getRow());
313     }
314   }
315 }
316 
317 @Test
318 public void testGetsWithLabels() throws Exception {
319   ThriftHBaseServiceHandler handler = createHandler();
320   byte[] rowName = "testPutGet".getBytes();
321   ByteBuffer table = wrap(tableAname);
322 
323   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
324   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
325       wrap(valueAname)));
326   columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
327       wrap(valueBname)));
328   TPut put = new TPut(wrap(rowName), columnValues);
329 
330   put.setColumnValues(columnValues);
331   put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
332       + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
333   handler.put(table, put);
334   TGet get = new TGet(wrap(rowName));
335   TAuthorization tauth = new TAuthorization();
336   List<String> labels = new ArrayList<String>();
337   labels.add(SECRET);
338   labels.add(PRIVATE);
339   tauth.setLabels(labels);
340   get.setAuthorizations(tauth);
341   TResult result = handler.get(table, get);
342   assertArrayEquals(rowName, result.getRow());
343   List<TColumnValue> returnedColumnValues = result.getColumnValues();
344   assertTColumnValuesEqual(columnValues, returnedColumnValues);
345 }
346 
347 @Test
348 public void testIncrementWithTags() throws Exception {
349   ThriftHBaseServiceHandler handler = createHandler();
350   byte[] rowName = "testIncrementWithTags".getBytes();
351   ByteBuffer table = wrap(tableAname);
352 
353   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
354   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
355       wrap(Bytes.toBytes(1L))));
356   TPut put = new TPut(wrap(rowName), columnValues);
357   put.setColumnValues(columnValues);
358   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
359   handler.put(table, put);
360 
361   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
362   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
363       wrap(qualifierAname)));
364   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
365   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
366   handler.increment(table, increment);
367 
368   TGet get = new TGet(wrap(rowName));
369   TAuthorization tauth = new TAuthorization();
370   List<String> labels = new ArrayList<String>();
371   labels.add(SECRET);
372   tauth.setLabels(labels);
373   get.setAuthorizations(tauth);
374   TResult result = handler.get(table, get);
375 
376   assertArrayEquals(rowName, result.getRow());
377   assertEquals(1, result.getColumnValuesSize());
378   TColumnValue columnValue = result.getColumnValues().get(0);
379   assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
380 }
381 
382 @Test
383 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
384   ThriftHBaseServiceHandler handler = createHandler();
385   byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
386   ByteBuffer table = wrap(tableAname);
387 
388   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
389   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
390       wrap(Bytes.toBytes(1L))));
391   TPut put = new TPut(wrap(rowName), columnValues);
392   put.setColumnValues(columnValues);
393   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
394   handler.put(table, put);
395 
396   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
397   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
398       wrap(qualifierAname)));
399   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
400   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
401   handler.increment(table, increment);
402 
403   TGet get = new TGet(wrap(rowName));
404   TAuthorization tauth = new TAuthorization();
405   List<String> labels = new ArrayList<String>();
406   labels.add(PUBLIC);
407   tauth.setLabels(labels);
408   get.setAuthorizations(tauth);
409   TResult result = handler.get(table, get);
410   assertNull(result.getRow());
411 }
412 
413 @Test
414 public void testAppend() throws Exception {
415   ThriftHBaseServiceHandler handler = createHandler();
416   byte[] rowName = "testAppend".getBytes();
417   ByteBuffer table = wrap(tableAname);
418   byte[] v1 = Bytes.toBytes(1L);
419   byte[] v2 = Bytes.toBytes(5L);
420   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
421   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
422       wrap(Bytes.toBytes(1L))));
423   TPut put = new TPut(wrap(rowName), columnValues);
424   put.setColumnValues(columnValues);
425   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
426   handler.put(table, put);
427 
428   List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
429   appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
430       wrap(v2)));
431   TAppend append = new TAppend(wrap(rowName), appendColumns);
432   append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
433   handler.append(table, append);
434 
435   TGet get = new TGet(wrap(rowName));
436   TAuthorization tauth = new TAuthorization();
437   List<String> labels = new ArrayList<String>();
438   labels.add(SECRET);
439   tauth.setLabels(labels);
440   get.setAuthorizations(tauth);
441   TResult result = handler.get(table, get);
442 
443   assertArrayEquals(rowName, result.getRow());
444   assertEquals(1, result.getColumnValuesSize());
445   TColumnValue columnValue = result.getColumnValues().get(0);
446   assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
447 }
448 
449 /**
450  * Padding numbers to make comparison of sort order easier in a for loop
451  * 
452  * @param n
453  *          The number to pad.
454  * @param pad
455  *          The length to pad up to.
456  * @return The padded number as a string.
457  */
458 private String pad(int n, byte pad) {
459   String res = Integer.toString(n);
460   while (res.length() < pad)
461     res = "0" + res;
462   return res;
463 }
464 }