View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataInputStream;
24  import java.io.DataOutputStream;
25  import java.io.IOException;
26  import java.util.Arrays;
27  import java.util.Collections;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import junit.framework.TestCase;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.KeyValue.KVComparator;
38  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
39  import org.apache.hadoop.hbase.KeyValue.Type;
40  import org.apache.hadoop.hbase.util.Bytes;
41  
42  import static org.junit.Assert.assertNotEquals;
43  
44  public class TestKeyValue extends TestCase {
45    private static final Log LOG = LogFactory.getLog(TestKeyValue.class);
46  
47    public void testColumnCompare() throws Exception {
48      final byte [] a = Bytes.toBytes("aaa");
49      byte [] family1 = Bytes.toBytes("abc");
50      byte [] qualifier1 = Bytes.toBytes("def");
51      byte [] family2 = Bytes.toBytes("abcd");
52      byte [] qualifier2 = Bytes.toBytes("ef");
53  
54      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a);
55      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
56      assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
57      aaa = new KeyValue(a, family2, qualifier2, 0L, Type.Put, a);
58      assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
59      assertTrue(CellUtil.matchingColumn(aaa, family2,qualifier2));
60      byte [] nullQualifier = new byte[0];
61      aaa = new KeyValue(a, family1, nullQualifier, 0L, Type.Put, a);
62      assertTrue(CellUtil.matchingColumn(aaa, family1,null));
63      assertFalse(CellUtil.matchingColumn(aaa, family2,qualifier2));
64    }
65  
66    /**
67     * Test a corner case when the family qualifier is a prefix of the
68     *  column qualifier.
69     */
70    public void testColumnCompare_prefix() throws Exception {
71      final byte [] a = Bytes.toBytes("aaa");
72      byte [] family1 = Bytes.toBytes("abc");
73      byte [] qualifier1 = Bytes.toBytes("def");
74      byte [] family2 = Bytes.toBytes("ab");
75      byte [] qualifier2 = Bytes.toBytes("def");
76  
77      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a);
78      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
79    }
80  
81    public void testBasics() throws Exception {
82      LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
83      check(Bytes.toBytes(getName()),
84        Bytes.toBytes(getName()), Bytes.toBytes(getName()), 1,
85        Bytes.toBytes(getName()));
86      // Test empty value and empty column -- both should work. (not empty fam)
87      check(Bytes.toBytes(getName()), Bytes.toBytes(getName()), null, 1, null);
88      check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(getName()), null, 1, null);
89      // empty qual is equivalent to null qual
90      assertEquals(
91        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
92        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"),
93          HConstants.EMPTY_BYTE_ARRAY, 1, (byte[]) null));
94    }
95  
96    private void check(final byte [] row, final byte [] family, byte [] qualifier,
97      final long timestamp, final byte [] value) {
98      KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
99      assertTrue(Bytes.compareTo(kv.getRow(), row) == 0);
100     assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
101     // Call toString to make sure it works.
102     LOG.info(kv.toString());
103   }
104 
105   public void testPlainCompare() throws Exception {
106     final byte [] a = Bytes.toBytes("aaa");
107     final byte [] b = Bytes.toBytes("bbb");
108     final byte [] fam = Bytes.toBytes("col");
109     final byte [] qf = Bytes.toBytes("umn");
110     KeyValue aaa = new KeyValue(a, fam, qf, a);
111     KeyValue bbb = new KeyValue(b, fam, qf, b);
112     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
113     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
114     // Compare breaks if passed same ByteBuffer as both left and right arguments.
115     assertTrue(KeyValue.COMPARATOR.compare(bbb, bbb) == 0);
116     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
117     // Do compare with different timestamps.
118     aaa = new KeyValue(a, fam, qf, 1, a);
119     bbb = new KeyValue(a, fam, qf, 2, a);
120     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) > 0);
121     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) < 0);
122     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
123     // Do compare with different types.  Higher numbered types -- Delete
124     // should sort ahead of lower numbers; i.e. Put
125     aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
126     bbb = new KeyValue(a, fam, qf, 1, a);
127     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
128     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
129     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
130   }
131 
132   public void testMoreComparisons() throws Exception {
133     long now = System.currentTimeMillis();
134 
135     // Meta compares
136     KeyValue aaa = new KeyValue(
137         Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
138     KeyValue bbb = new KeyValue(
139         Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
140     KVComparator c = new KeyValue.MetaComparator();
141     assertTrue(c.compare(bbb, aaa) < 0);
142 
143     KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
144         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L,
145         (byte[])null);
146     assertTrue(c.compare(aaaa, bbb) < 0);
147 
148     KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
149         Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L,
150         (byte[])null);
151     KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
152         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L,
153         (byte[])null);
154     assertTrue(c.compare(x, y) < 0);
155     comparisons(new KeyValue.MetaComparator());
156     comparisons(new KeyValue.KVComparator());
157     metacomparisons(new KeyValue.MetaComparator());
158   }
159 
160   public void testMetaComparatorTableKeysWithCommaOk() {
161     MetaComparator c = new KeyValue.MetaComparator();
162     long now = System.currentTimeMillis();
163     // meta keys values are not quite right.  A users can enter illegal values
164     // from shell when scanning meta.
165     KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
166     KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
167     assertTrue(c.compare(a, b) < 0);
168   }
169 
170   /**
171    * Tests cases where rows keys have characters below the ','.
172    * See HBASE-832
173    * @throws IOException
174    */
175   public void testKeyValueBorderCases() throws IOException {
176     // % sorts before , so if we don't do special comparator, rowB would
177     // come before rowA.
178     KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
179       Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
180     KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
181         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
182     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
183 
184     rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"),
185         Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
186     rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"),
187         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
188     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
189 
190   }
191 
192   private void metacomparisons(final KeyValue.MetaComparator c) {
193     long now = System.currentTimeMillis();
194     assertTrue(c.compare(new KeyValue(
195         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now),
196       new KeyValue(
197           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) == 0);
198     KeyValue a = new KeyValue(
199         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now);
200     KeyValue b = new KeyValue(
201         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now);
202     assertTrue(c.compare(a, b) < 0);
203     assertTrue(c.compare(new KeyValue(
204         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now),
205       new KeyValue(
206           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) > 0);
207   }
208 
209   private void comparisons(final KeyValue.KVComparator c) {
210     long now = System.currentTimeMillis();
211     assertTrue(c.compare(new KeyValue(
212         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
213       new KeyValue(
214           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) == 0);
215     assertTrue(c.compare(new KeyValue(
216         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
217       new KeyValue(
218           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now)) < 0);
219     assertTrue(c.compare(new KeyValue(
220         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now),
221       new KeyValue(
222           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) > 0);
223   }
224 
225   public void testBinaryKeys() throws Exception {
226     Set<KeyValue> set = new TreeSet<KeyValue>(KeyValue.COMPARATOR);
227     final byte [] fam = Bytes.toBytes("col");
228     final byte [] qf = Bytes.toBytes("umn");
229     final byte [] nb = new byte[0];
230     KeyValue [] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
231       new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
232       new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
233       new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
234       new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
235       new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb),
236     };
237     // Add to set with bad comparator
238     Collections.addAll(set, keys);
239     // This will output the keys incorrectly.
240     boolean assertion = false;
241     int count = 0;
242     try {
243       for (KeyValue k: set) {
244         assertTrue(count++ == k.getTimestamp());
245       }
246     } catch (junit.framework.AssertionFailedError e) {
247       // Expected
248       assertion = true;
249     }
250     assertTrue(assertion);
251     // Make set with good comparator
252     set = new TreeSet<KeyValue>(new KeyValue.MetaComparator());
253     Collections.addAll(set, keys);
254     count = 0;
255     for (KeyValue k: set) {
256       assertTrue(count++ == k.getTimestamp());
257     }
258   }
259 
260   public void testStackedUpKeyValue() {
261     // Test multiple KeyValues in a single blob.
262 
263     // TODO actually write this test!
264 
265   }
266 
267   private final byte[] rowA = Bytes.toBytes("rowA");
268   private final byte[] rowB = Bytes.toBytes("rowB");
269 
270   private final byte[] family = Bytes.toBytes("family");
271   private final byte[] qualA = Bytes.toBytes("qfA");
272   private final byte[] qualB = Bytes.toBytes("qfB");
273 
274   private void assertKVLess(KeyValue.KVComparator c,
275                             KeyValue less,
276                             KeyValue greater) {
277     int cmp = c.compare(less,greater);
278     assertTrue(cmp < 0);
279     cmp = c.compare(greater,less);
280     assertTrue(cmp > 0);
281   }
282 
283   private void assertKVLessWithoutRow(KeyValue.KVComparator c, int common, KeyValue less,
284       KeyValue greater) {
285     int cmp = c.compareIgnoringPrefix(common, less.getBuffer(), less.getOffset()
286         + KeyValue.ROW_OFFSET, less.getKeyLength(), greater.getBuffer(),
287         greater.getOffset() + KeyValue.ROW_OFFSET, greater.getKeyLength());
288     assertTrue(cmp < 0);
289     cmp = c.compareIgnoringPrefix(common, greater.getBuffer(), greater.getOffset()
290         + KeyValue.ROW_OFFSET, greater.getKeyLength(), less.getBuffer(),
291         less.getOffset() + KeyValue.ROW_OFFSET, less.getKeyLength());
292     assertTrue(cmp > 0);
293   }
294 
295   public void testCompareWithoutRow() {
296     final KeyValue.KVComparator c = KeyValue.COMPARATOR;
297     byte[] row = Bytes.toBytes("row");
298 
299     byte[] fa = Bytes.toBytes("fa");
300     byte[] fami = Bytes.toBytes("fami");
301     byte[] fami1 = Bytes.toBytes("fami1");
302 
303     byte[] qual0 = Bytes.toBytes("");
304     byte[] qual1 = Bytes.toBytes("qf1");
305     byte[] qual2 = Bytes.toBytes("qf2");
306     long ts = 1;
307 
308     // 'fa:'
309     KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, Type.Put);
310     // 'fami:'
311     KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, Type.Put);
312     // 'fami:qf1'
313     KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, Type.Put);
314     // 'fami:qf2'
315     KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, Type.Put);
316     // 'fami1:'
317     KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, Type.Put);
318 
319     // 'fami:qf1' < 'fami:qf2'
320     assertKVLessWithoutRow(c, 0, kv0_1, kv0_2);
321     // 'fami:qf1' < 'fami1:'
322     assertKVLessWithoutRow(c, 0, kv0_1, kv1_0);
323 
324     // Test comparison by skipping the same prefix bytes.
325     /***
326      * KeyValue Format and commonLength:
327      * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
328      * ------------------|-------commonLength--------|--------------
329      */
330     int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
331         + row.length;
332     // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
333     assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0);
334     // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
335     assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1);
336     // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
337     assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0);
338     // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
339     assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2);
340   }
341 
342   public void testFirstLastOnRow() {
343     final KVComparator c = KeyValue.COMPARATOR;
344     long ts = 1;
345     byte[] bufferA = new byte[128];
346     int offsetA = 0;
347     byte[] bufferB = new byte[128];
348     int offsetB = 7;
349 
350     // These are listed in sort order (ie: every one should be less
351     // than the one on the next line).
352     final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
353     final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA,
354         rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
355     final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put);
356     final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put);
357 
358     final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
359     final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
360     final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB,
361         rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
362     final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put);
363 
364     assertKVLess(c, firstOnRowA, firstOnRowB);
365     assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
366     assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
367     assertKVLess(c, firstOnRowA, kvA_1);
368     assertKVLess(c, firstOnRowA, kvA_2);
369     assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
370     assertKVLess(c, kvA_1, kvA_2);
371     assertKVLess(c, kvA_2, firstOnRowB);
372     assertKVLess(c, kvA_1, firstOnRowB);
373     assertKVLess(c, kvA_2, firstOnRowBBufferFam);
374     assertKVLess(c, kvA_1, firstOnRowBBufferFam);
375 
376     assertKVLess(c, lastOnRowA, firstOnRowB);
377     assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
378     assertKVLess(c, firstOnRowB, kvB);
379     assertKVLess(c, firstOnRowBBufferFam, kvB);
380     assertKVLess(c, lastOnRowA, kvB);
381 
382     assertKVLess(c, kvA_2, lastOnRowA);
383     assertKVLess(c, kvA_1, lastOnRowA);
384     assertKVLess(c, firstOnRowA, lastOnRowA);
385     assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
386   }
387 
388   public void testCreateKeyOnly() throws Exception {
389     long ts = 1;
390     byte [] value = Bytes.toBytes("a real value");
391     byte [] evalue = new byte[0]; // empty value
392 
393     for (byte[] val : new byte[][]{value, evalue}) {
394       for (boolean useLen : new boolean[]{false,true}) {
395         KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
396         KeyValue kv1ko = kv1.createKeyOnly(useLen);
397         // keys are still the same
398         assertTrue(kv1.equals(kv1ko));
399         // but values are not
400         assertTrue(kv1ko.getValue().length == (useLen?Bytes.SIZEOF_INT:0));
401         if (useLen) {
402           assertEquals(kv1.getValueLength(), Bytes.toInt(kv1ko.getValue()));
403         }
404       }
405     }
406   }
407 
408   public void testCreateKeyValueFromKey() {
409     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
410         Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
411     int initialPadding = 10;
412     int endingPadding = 20;
413     int keyLen = kv.getKeyLength();
414     byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
415     System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr,
416         initialPadding, keyLen);
417     KeyValue kvFromKey = KeyValue.createKeyValueFromKey(tmpArr, initialPadding,
418         keyLen);
419     assertEquals(keyLen, kvFromKey.getKeyLength());
420     assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
421     System.err.println("kv=" + kv);
422     System.err.println("kvFromKey=" + kvFromKey);
423     assertEquals(kvFromKey.toString(),
424         kv.toString().replaceAll("=[0-9]+", "=0"));
425   }
426 
427   /**
428    * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
429    * See HBASE-6265.
430    */
431   public void testGetTimestamp() {
432     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
433       Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
434       Bytes.toBytes("myValue"));
435     long time1 = kv.getTimestamp();
436     kv.updateLatestStamp(Bytes.toBytes(12345L));
437     long time2 = kv.getTimestamp();
438     assertEquals(HConstants.LATEST_TIMESTAMP, time1);
439     assertEquals(12345L, time2);
440   }
441 
442   /**
443    * See HBASE-7845
444    */
445   public void testGetShortMidpointKey() {
446     final KVComparator keyComparator = KeyValue.COMPARATOR;
447     //verify that faked shorter rowkey could be generated
448     long ts = 5;
449     KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put);
450     KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put);
451     byte[] newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
452     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
453     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
454     short newRowLength = Bytes.toShort(newKey, 0);
455     byte[] expectedArray = Bytes.toBytes("the r");
456     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
457       expectedArray.length);
458 
459     //verify: same with "row + family + qualifier", return rightKey directly
460     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
461     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put);
462     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
463     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
464     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
465     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
466     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put);
467     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put);
468     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
469     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
470     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
471     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
472 
473     // verify: same with row, different with qualifier
474     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put);
475     kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put);
476     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
477     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
478     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
479     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
480     KeyValue newKeyValue = KeyValue.createKeyValueFromKey(newKey);
481     assertTrue(Arrays.equals(newKeyValue.getFamily(),family));
482     assertTrue(Arrays.equals(newKeyValue.getQualifier(),qualB));
483     assertTrue(newKeyValue.getTimestamp() == HConstants.LATEST_TIMESTAMP);
484     assertTrue(newKeyValue.getTypeByte() == Type.Maximum.getCode());
485 
486     //verify metaKeyComparator's getShortMidpointKey output
487     final KVComparator metaKeyComparator = KeyValue.META_COMPARATOR;
488     kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put);
489     kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put);
490     newKey = metaKeyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
491     assertTrue(metaKeyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
492     assertTrue(metaKeyComparator.compareFlatKey(newKey, kv2.getKey()) == 0);
493 
494     //verify common fix scenario
495     kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put);
496     kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put);
497     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
498     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
499     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
500     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
501     newRowLength = Bytes.toShort(newKey, 0);
502     expectedArray = Bytes.toBytes("ilovehbasea");
503     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
504       expectedArray.length);
505     //verify only 1 offset scenario
506     kv1 = new KeyValue(Bytes.toBytes("100abcdefg"), family, qualA, ts, Type.Put);
507     kv2 = new KeyValue(Bytes.toBytes("101abcdefg"), family, qualA, ts, Type.Put);
508     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0);
509     newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey());
510     assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0);
511     assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0);
512     newRowLength = Bytes.toShort(newKey, 0);
513     expectedArray = Bytes.toBytes("101");
514     Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0,
515       expectedArray.length);
516   }
517 
518   public void testKVsWithTags() {
519     byte[] row = Bytes.toBytes("myRow");
520     byte[] cf = Bytes.toBytes("myCF");
521     byte[] q = Bytes.toBytes("myQualifier");
522     byte[] value = Bytes.toBytes("myValue");
523     byte[] metaValue1 = Bytes.toBytes("metaValue1");
524     byte[] metaValue2 = Bytes.toBytes("metaValue2");
525     KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
526         new Tag((byte) 1, metaValue1), new Tag((byte) 2, metaValue2) });
527     assertTrue(kv.getTagsLength() > 0);
528     assertTrue(Bytes.equals(kv.getRow(), row));
529     assertTrue(Bytes.equals(kv.getFamily(), cf));
530     assertTrue(Bytes.equals(kv.getQualifier(), q));
531     assertTrue(Bytes.equals(kv.getValue(), value));
532     List<Tag> tags = kv.getTags();
533     assertNotNull(tags);
534     assertEquals(2, tags.size());
535     boolean meta1Ok = false, meta2Ok = false;
536     for (Tag tag : tags) {
537       if (tag.getType() == (byte) 1) {
538         if (Bytes.equals(tag.getValue(), metaValue1)) {
539           meta1Ok = true;
540         }
541       } else {
542         if (Bytes.equals(tag.getValue(), metaValue2)) {
543           meta2Ok = true;
544         }
545       }
546     }
547     assertTrue(meta1Ok);
548     assertTrue(meta2Ok);
549     Iterator<Tag> tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
550         kv.getTagsLength());
551     //Iterator<Tag> tagItr = kv.tagsIterator();
552     assertTrue(tagItr.hasNext());
553     Tag next = tagItr.next();
554     assertEquals(10, next.getTagLength());
555     assertEquals((byte) 1, next.getType());
556     Bytes.equals(next.getValue(), metaValue1);
557     assertTrue(tagItr.hasNext());
558     next = tagItr.next();
559     assertEquals(10, next.getTagLength());
560     assertEquals((byte) 2, next.getType());
561     Bytes.equals(next.getValue(), metaValue2);
562     assertFalse(tagItr.hasNext());
563 
564     tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
565         kv.getTagsLength());
566     assertTrue(tagItr.hasNext());
567     next = tagItr.next();
568     assertEquals(10, next.getTagLength());
569     assertEquals((byte) 1, next.getType());
570     Bytes.equals(next.getValue(), metaValue1);
571     assertTrue(tagItr.hasNext());
572     next = tagItr.next();
573     assertEquals(10, next.getTagLength());
574     assertEquals((byte) 2, next.getType());
575     Bytes.equals(next.getValue(), metaValue2);
576     assertFalse(tagItr.hasNext());
577   }
578   
579   public void testMetaKeyComparator() {
580     MetaComparator c = new KeyValue.MetaComparator();
581     long now = System.currentTimeMillis();
582 
583     KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
584     KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
585     assertTrue(c.compare(a, b) < 0);
586     
587     a = new KeyValue(Bytes.toBytes("table1,111"), now);
588     b = new KeyValue(Bytes.toBytes("table2"), now);
589     assertTrue(c.compare(a, b) < 0);
590     
591     a = new KeyValue(Bytes.toBytes("table1"), now);
592     b = new KeyValue(Bytes.toBytes("table2,111"), now);
593     assertTrue(c.compare(a, b) < 0);
594     
595     a = new KeyValue(Bytes.toBytes("table,111"), now);
596     b = new KeyValue(Bytes.toBytes("table,2222"), now);
597     assertTrue(c.compare(a, b) < 0);
598     
599     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
600     b = new KeyValue(Bytes.toBytes("table,2222"), now);
601     assertTrue(c.compare(a, b) < 0);
602     
603     a = new KeyValue(Bytes.toBytes("table,111"), now);
604     b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
605     assertTrue(c.compare(a, b) < 0);
606 
607     a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
608     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
609     assertTrue(c.compare(a, b) < 0);
610     
611     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
612     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
613     assertTrue(c.compare(a, b) < 0);
614 
615     a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
616     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
617     assertTrue(c.compare(a, b) < 0);
618     
619     a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
620     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
621     assertTrue(c.compare(a, b) < 0);
622   }
623 
624   public void testKeyValueSerialization() throws Exception {
625     KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
626         Bytes.toBytes("1"));
627     KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
628         Bytes.toBytes("2"));
629     MockKeyValue mkvA1 = new MockKeyValue(kvA1);
630     MockKeyValue mkvA2 = new MockKeyValue(kvA2);
631     ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
632     DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
633     KeyValueUtil.oswrite(mkvA1, os, true);
634     KeyValueUtil.oswrite(mkvA2, os, true);
635     DataInputStream is = new DataInputStream(new ByteArrayInputStream(
636         byteArrayOutputStream.toByteArray()));
637     KeyValue deSerKV1 = KeyValue.iscreate(is);
638     assertTrue(kvA1.equals(deSerKV1));
639     KeyValue deSerKV2 = KeyValue.iscreate(is);
640     assertTrue(kvA2.equals(deSerKV2));
641   }
642 
643   private class MockKeyValue implements Cell {
644     private final KeyValue kv;
645 
646     public MockKeyValue(KeyValue kv) {
647       this.kv = kv;
648     }
649 
650     /**
651      * This returns the offset where the tag actually starts.
652      */
653     @Override
654     public int getTagsOffset() {
655       return this.kv.getTagsOffset();
656     }
657 
658     // used to achieve atomic operations in the memstore.
659     @Override
660     public long getMvccVersion() {
661       return this.kv.getMvccVersion();
662     }
663 
664     /**
665      * used to achieve atomic operations in the memstore.
666      */
667     @Override
668     public long getSequenceId() {
669       return this.kv.getSequenceId();
670     }
671 
672     /**
673      * This returns the total length of the tag bytes
674      */
675     @Override
676     public int getTagsLength() {
677       return this.kv.getTagsLength();
678     }
679 
680     /**
681      * 
682      * @return Timestamp
683      */
684     @Override
685     public long getTimestamp() {
686       return this.kv.getTimestamp();
687     }
688 
689     /**
690      * @return KeyValue.TYPE byte representation
691      */
692     @Override
693     public byte getTypeByte() {
694       return this.kv.getTypeByte();
695     }
696 
697     /**
698      * @return the backing array of the entire KeyValue (all KeyValue fields are
699      *         in a single array)
700      */
701     @Override
702     public byte[] getValueArray() {
703       return this.kv.getValueArray();
704     }
705 
706     /**
707      * @return the value offset
708      */
709     @Override
710     public int getValueOffset() {
711       return this.kv.getValueOffset();
712     }
713 
714     /**
715      * @return Value length
716      */
717     @Override
718     public int getValueLength() {
719       return this.kv.getValueLength();
720     }
721 
722     /**
723      * @return the backing array of the entire KeyValue (all KeyValue fields are
724      *         in a single array)
725      */
726     @Override
727     public byte[] getRowArray() {
728       return this.kv.getRowArray();
729     }
730 
731     /**
732      * @return Row offset
733      */
734     @Override
735     public int getRowOffset() {
736       return this.kv.getRowOffset();
737     }
738 
739     /**
740      * @return Row length
741      */
742     @Override
743     public short getRowLength() {
744       return this.kv.getRowLength();
745     }
746 
747     /**
748      * @return the backing array of the entire KeyValue (all KeyValue fields are
749      *         in a single array)
750      */
751     @Override
752     public byte[] getFamilyArray() {
753       return this.kv.getFamilyArray();
754     }
755 
756     /**
757      * @return Family offset
758      */
759     @Override
760     public int getFamilyOffset() {
761       return this.kv.getFamilyOffset();
762     }
763 
764     /**
765      * @return Family length
766      */
767     @Override
768     public byte getFamilyLength() {
769       return this.kv.getFamilyLength();
770     }
771 
772     /**
773      * @return the backing array of the entire KeyValue (all KeyValue fields are
774      *         in a single array)
775      */
776     @Override
777     public byte[] getQualifierArray() {
778       return this.kv.getQualifierArray();
779     }
780 
781     /**
782      * @return Qualifier offset
783      */
784     @Override
785     public int getQualifierOffset() {
786       return this.kv.getQualifierOffset();
787     }
788 
789     /**
790      * @return Qualifier length
791      */
792     @Override
793     public int getQualifierLength() {
794       return this.kv.getQualifierLength();
795     }
796 
797     @Override
798     @Deprecated
799     public byte[] getValue() {
800       // TODO Auto-generated method stub
801       return null;
802     }
803 
804     @Override
805     @Deprecated
806     public byte[] getFamily() {
807       // TODO Auto-generated method stub
808       return null;
809     }
810 
811     @Override
812     @Deprecated
813     public byte[] getQualifier() {
814       // TODO Auto-generated method stub
815       return null;
816     }
817 
818     @Override
819     @Deprecated
820     public byte[] getRow() {
821       // TODO Auto-generated method stub
822       return null;
823     }
824 
825     /**
826      * @return the backing array of the entire KeyValue (all KeyValue fields are
827      *         in a single array)
828      */
829     @Override
830     public byte[] getTagsArray() {
831       return this.kv.getTagsArray();
832     }
833   }
834 
835   public void testEqualsAndHashCode() throws Exception {
836     KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
837         Bytes.toBytes("qualA"), Bytes.toBytes("1"));
838     KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
839         Bytes.toBytes("qualA"), Bytes.toBytes("2"));
840     // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
841     // don't take this into account.
842     kvA2.setSequenceId(2);
843     KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
844         Bytes.toBytes("qualB"), Bytes.toBytes("1"));
845 
846     assertEquals(kvA1, kvA2);
847     assertNotEquals(kvA1, kvB);
848     assertEquals(kvA1.hashCode(), kvA2.hashCode());
849     assertNotEquals(kvA1.hashCode(), kvB.hashCode());
850   }
851 
852 }