View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.List;
29  import java.util.Random;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.client.Admin;
35  import org.apache.hadoop.hbase.client.Connection;
36  import org.apache.hadoop.hbase.client.ConnectionFactory;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.RegionLocator;
40  import org.apache.hadoop.hbase.client.Result;
41  import org.apache.hadoop.hbase.client.Table;
42  import org.apache.hadoop.hbase.ipc.CallRunner;
43  import org.apache.hadoop.hbase.ipc.DelegatingRpcScheduler;
44  import org.apache.hadoop.hbase.ipc.PriorityFunction;
45  import org.apache.hadoop.hbase.ipc.RpcScheduler;
46  import org.apache.hadoop.hbase.regionserver.HRegionServer;
47  import org.apache.hadoop.hbase.regionserver.RSRpcServices;
48  import org.apache.hadoop.hbase.regionserver.SimpleRpcSchedulerFactory;
49  import org.apache.hadoop.hbase.testclassification.MediumTests;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
52  import org.apache.hadoop.hbase.util.Pair;
53  import org.junit.AfterClass;
54  import org.junit.Assert;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.experimental.categories.Category;
58  
59  import com.google.common.collect.Lists;
60  
61  /**
62   * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}.
63   */
64  @Category(MediumTests.class)
65  public class TestMetaTableAccessor {
66    private static final Log LOG = LogFactory.getLog(TestMetaTableAccessor.class);
67    private static final  HBaseTestingUtility UTIL = new HBaseTestingUtility();
68    private static Connection connection;
69    private Random random = new Random();
70  
71    @BeforeClass public static void beforeClass() throws Exception {
72      UTIL.startMiniCluster(3);
73  
74      Configuration c = new Configuration(UTIL.getConfiguration());
75      // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
76      // responsive.  1 second is default as is ten retries.
77      c.setLong("hbase.client.pause", 1000);
78      c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
79      connection = ConnectionFactory.createConnection(c);
80    }
81  
82    @AfterClass public static void afterClass() throws Exception {
83      connection.close();
84      UTIL.shutdownMiniCluster();
85    }
86  
87    /**
88     * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write
89     * against hbase:meta while its hosted server is restarted to prove our retrying
90     * works.
91     * @throws IOException
92     * @throws InterruptedException
93     */
94    @Test public void testRetrying()
95    throws IOException, InterruptedException {
96      final TableName name =
97          TableName.valueOf("testRetrying");
98      LOG.info("Started " + name);
99      HTable t = UTIL.createMultiRegionTable(name, HConstants.CATALOG_FAMILY);
100     int regionCount = -1;
101     try (RegionLocator r = t.getRegionLocator()) {
102       regionCount = r.getStartKeys().length;
103     }
104     // Test it works getting a region from just made user table.
105     final List<HRegionInfo> regions =
106       testGettingTableRegions(connection, name, regionCount);
107     MetaTask reader = new MetaTask(connection, "reader") {
108       @Override
109       void metaTask() throws Throwable {
110         testGetRegion(connection, regions.get(0));
111         LOG.info("Read " + regions.get(0).getEncodedName());
112       }
113     };
114     MetaTask writer = new MetaTask(connection, "writer") {
115       @Override
116       void metaTask() throws Throwable {
117         MetaTableAccessor.addRegionToMeta(connection, regions.get(0));
118         LOG.info("Wrote " + regions.get(0).getEncodedName());
119       }
120     };
121     reader.start();
122     writer.start();
123 
124     // We're gonna check how it takes. If it takes too long, we will consider
125     //  it as a fail. We can't put that in the @Test tag as we want to close
126     //  the threads nicely
127     final long timeOut = 180000;
128     long startTime = System.currentTimeMillis();
129 
130     try {
131       // Make sure reader and writer are working.
132       assertTrue(reader.isProgressing());
133       assertTrue(writer.isProgressing());
134 
135       // Kill server hosting meta -- twice  . See if our reader/writer ride over the
136       // meta moves.  They'll need to retry.
137       for (int i = 0; i < 2; i++) {
138         LOG.info("Restart=" + i);
139         UTIL.ensureSomeRegionServersAvailable(2);
140         int index = -1;
141         do {
142           index = UTIL.getMiniHBaseCluster().getServerWithMeta();
143         } while (index == -1 &&
144           startTime + timeOut < System.currentTimeMillis());
145 
146         if (index != -1){
147           UTIL.getMiniHBaseCluster().abortRegionServer(index);
148           UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
149         }
150       }
151 
152       assertTrue("reader: " + reader.toString(), reader.isProgressing());
153       assertTrue("writer: " + writer.toString(), writer.isProgressing());
154     } catch (IOException e) {
155       throw e;
156     } finally {
157       reader.stop = true;
158       writer.stop = true;
159       reader.join();
160       writer.join();
161       t.close();
162     }
163     long exeTime = System.currentTimeMillis() - startTime;
164     assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
165   }
166 
167   /**
168    * Thread that runs a MetaTableAccessor task until asked stop.
169    */
170   abstract static class MetaTask extends Thread {
171     boolean stop = false;
172     int count = 0;
173     Throwable t = null;
174     final Connection connection;
175 
176     MetaTask(final Connection connection, final String name) {
177       super(name);
178       this.connection = connection;
179     }
180 
181     @Override
182     public void run() {
183       try {
184         while(!this.stop) {
185           LOG.info("Before " + this.getName()+ ", count=" + this.count);
186           metaTask();
187           this.count += 1;
188           LOG.info("After " + this.getName() + ", count=" + this.count);
189           Thread.sleep(100);
190         }
191       } catch (Throwable t) {
192         LOG.info(this.getName() + " failed", t);
193         this.t = t;
194       }
195     }
196 
197     boolean isProgressing() throws InterruptedException {
198       int currentCount = this.count;
199       while(currentCount == this.count) {
200         if (!isAlive()) return false;
201         if (this.t != null) return false;
202         Thread.sleep(10);
203       }
204       return true;
205     }
206 
207     @Override
208     public String toString() {
209       return "count=" + this.count + ", t=" +
210         (this.t == null? "null": this.t.toString());
211     }
212 
213     abstract void metaTask() throws Throwable;
214   }
215 
216   @Test public void testGetRegionsCatalogTables()
217   throws IOException, InterruptedException {
218     List<HRegionInfo> regions =
219       MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
220         connection, TableName.META_TABLE_NAME);
221     assertTrue(regions.size() >= 1);
222     assertTrue(MetaTableAccessor.getTableRegionsAndLocations(UTIL.getZooKeeperWatcher(),
223       connection,TableName.META_TABLE_NAME).size() >= 1);
224   }
225 
226   @Test public void testTableExists() throws IOException {
227     final TableName name =
228         TableName.valueOf("testTableExists");
229     assertFalse(MetaTableAccessor.tableExists(connection, name));
230     UTIL.createTable(name, HConstants.CATALOG_FAMILY);
231     assertTrue(MetaTableAccessor.tableExists(connection, name));
232     Admin admin = UTIL.getHBaseAdmin();
233     admin.disableTable(name);
234     admin.deleteTable(name);
235     assertFalse(MetaTableAccessor.tableExists(connection, name));
236     assertTrue(MetaTableAccessor.tableExists(connection,
237       TableName.META_TABLE_NAME));
238   }
239 
240   @Test public void testGetRegion() throws IOException, InterruptedException {
241     final String name = "testGetRegion";
242     LOG.info("Started " + name);
243     // Test get on non-existent region.
244     Pair<HRegionInfo, ServerName> pair =
245       MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
246     assertNull(pair);
247     LOG.info("Finished " + name);
248   }
249 
250   // Test for the optimization made in HBASE-3650
251   @Test public void testScanMetaForTable()
252   throws IOException, InterruptedException {
253     final TableName name =
254         TableName.valueOf("testScanMetaForTable");
255     LOG.info("Started " + name);
256 
257     /** Create 2 tables
258      - testScanMetaForTable
259      - testScanMetaForTablf
260     **/
261 
262     UTIL.createTable(name, HConstants.CATALOG_FAMILY);
263     // name that is +1 greater than the first one (e+1=f)
264     TableName greaterName =
265         TableName.valueOf("testScanMetaForTablf");
266     UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
267 
268     // Now make sure we only get the regions from 1 of the tables at a time
269 
270     assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
271       connection, name).size());
272     assertEquals(1, MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
273       connection, greaterName).size());
274   }
275 
276   private static List<HRegionInfo> testGettingTableRegions(final Connection connection,
277       final TableName name, final int regionCount)
278   throws IOException, InterruptedException {
279     List<HRegionInfo> regions = MetaTableAccessor.getTableRegions(UTIL.getZooKeeperWatcher(),
280       connection, name);
281     assertEquals(regionCount, regions.size());
282     Pair<HRegionInfo, ServerName> pair =
283       MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
284     assertEquals(regions.get(0).getEncodedName(),
285       pair.getFirst().getEncodedName());
286     return regions;
287   }
288 
289   private static void testGetRegion(final Connection connection,
290       final HRegionInfo region)
291   throws IOException, InterruptedException {
292     Pair<HRegionInfo, ServerName> pair =
293       MetaTableAccessor.getRegion(connection, region.getRegionName());
294     assertEquals(region.getEncodedName(),
295       pair.getFirst().getEncodedName());
296   }
297 
298   @Test
299   public void testParseReplicaIdFromServerColumn() {
300     String column1 = HConstants.SERVER_QUALIFIER_STR;
301     assertEquals(0, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column1)));
302     String column2 = column1 + MetaTableAccessor.META_REPLICA_ID_DELIMITER;
303     assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column2)));
304     String column3 = column2 + "00";
305     assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column3)));
306     String column4 = column3 + "2A";
307     assertEquals(42, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column4)));
308     String column5 = column4 + "2A";
309     assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column5)));
310     String column6 = HConstants.STARTCODE_QUALIFIER_STR;
311     assertEquals(-1, MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column6)));
312   }
313 
314   @Test
315   public void testMetaReaderGetColumnMethods() {
316     Assert.assertArrayEquals(HConstants.SERVER_QUALIFIER, MetaTableAccessor.getServerColumn(0));
317     Assert.assertArrayEquals(Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR
318       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
319       MetaTableAccessor.getServerColumn(42));
320 
321     Assert.assertArrayEquals(HConstants.STARTCODE_QUALIFIER,
322       MetaTableAccessor.getStartCodeColumn(0));
323     Assert.assertArrayEquals(Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR
324       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
325       MetaTableAccessor.getStartCodeColumn(42));
326 
327     Assert.assertArrayEquals(HConstants.SEQNUM_QUALIFIER,
328       MetaTableAccessor.getSeqNumColumn(0));
329     Assert.assertArrayEquals(Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR
330       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
331       MetaTableAccessor.getSeqNumColumn(42));
332   }
333 
334   @Test
335   public void testMetaLocationsForRegionReplicas() throws IOException {
336     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
337     ServerName serverName1 = ServerName.valueOf("bar", 60010, random.nextLong());
338     ServerName serverName100 = ServerName.valueOf("baz", 60010, random.nextLong());
339 
340     long regionId = System.currentTimeMillis();
341     HRegionInfo primary = new HRegionInfo(TableName.valueOf("table_foo"),
342       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
343     HRegionInfo replica1 = new HRegionInfo(TableName.valueOf("table_foo"),
344       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 1);
345     HRegionInfo replica100 = new HRegionInfo(TableName.valueOf("table_foo"),
346       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 100);
347 
348     long seqNum0 = random.nextLong();
349     long seqNum1 = random.nextLong();
350     long seqNum100 = random.nextLong();
351 
352 
353     Table meta = MetaTableAccessor.getMetaHTable(connection);
354     try {
355       MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0, -1);
356 
357       // assert that the server, startcode and seqNum columns are there for the primary region
358       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
359 
360       // add replica = 1
361       MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1, -1);
362       // check whether the primary is still there
363       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
364       // now check for replica 1
365       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
366 
367       // add replica = 1
368       MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100, -1);
369       // check whether the primary is still there
370       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
371       // check whether the replica 1 is still there
372       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
373       // now check for replica 1
374       assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
375     } finally {
376       meta.close();
377     }
378   }
379 
380   public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName,
381       long seqNum, int replicaId, boolean checkSeqNum) throws IOException {
382     Get get = new Get(row);
383     Result result = meta.get(get);
384     assertTrue(Bytes.equals(
385       result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(replicaId)),
386       Bytes.toBytes(serverName.getHostAndPort())));
387     assertTrue(Bytes.equals(
388       result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(replicaId)),
389       Bytes.toBytes(serverName.getStartcode())));
390     if (checkSeqNum) {
391       assertTrue(Bytes.equals(
392         result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(replicaId)),
393         Bytes.toBytes(seqNum)));
394     }
395   }
396 
397   public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId)
398       throws IOException {
399     Get get = new Get(row);
400     Result result = meta.get(get);
401     Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
402       MetaTableAccessor.getServerColumn(replicaId));
403     Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
404       MetaTableAccessor.getStartCodeColumn(replicaId));
405     assertNotNull(serverCell);
406     assertNotNull(startCodeCell);
407     assertEquals(0, serverCell.getValueLength());
408     assertEquals(0, startCodeCell.getValueLength());
409   }
410 
411   @Test
412   public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException {
413     long regionId = System.currentTimeMillis();
414     HRegionInfo primary = new HRegionInfo(TableName.valueOf("table_foo"),
415       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
416 
417     Table meta = MetaTableAccessor.getMetaHTable(connection);
418     try {
419       List<HRegionInfo> regionInfos = Lists.newArrayList(primary);
420       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
421 
422       assertEmptyMetaLocation(meta, primary.getRegionName(), 1);
423       assertEmptyMetaLocation(meta, primary.getRegionName(), 2);
424     } finally {
425       meta.close();
426     }
427   }
428 
429   @Test
430   public void testMetaLocationForRegionReplicasIsAddedAtRegionSplit() throws IOException {
431     long regionId = System.currentTimeMillis();
432     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
433     HRegionInfo parent = new HRegionInfo(TableName.valueOf("table_foo"),
434       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
435     HRegionInfo splitA = new HRegionInfo(TableName.valueOf("table_foo"),
436       HConstants.EMPTY_START_ROW, Bytes.toBytes("a"), false, regionId+1, 0);
437     HRegionInfo splitB = new HRegionInfo(TableName.valueOf("table_foo"),
438       Bytes.toBytes("a"), HConstants.EMPTY_END_ROW, false, regionId+1, 0);
439 
440 
441     Table meta = MetaTableAccessor.getMetaHTable(connection);
442     try {
443       List<HRegionInfo> regionInfos = Lists.newArrayList(parent);
444       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
445 
446       MetaTableAccessor.splitRegion(connection, parent, splitA, splitB, serverName0, 3);
447 
448       assertEmptyMetaLocation(meta, splitA.getRegionName(), 1);
449       assertEmptyMetaLocation(meta, splitA.getRegionName(), 2);
450       assertEmptyMetaLocation(meta, splitB.getRegionName(), 1);
451       assertEmptyMetaLocation(meta, splitB.getRegionName(), 2);
452     } finally {
453       meta.close();
454     }
455   }
456 
457   @Test
458   public void testMetaLocationForRegionReplicasIsAddedAtRegionMerge() throws IOException {
459     long regionId = System.currentTimeMillis();
460     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
461 
462     HRegionInfo parentA = new HRegionInfo(TableName.valueOf("table_foo"),
463       Bytes.toBytes("a"), HConstants.EMPTY_END_ROW, false, regionId, 0);
464     HRegionInfo parentB = new HRegionInfo(TableName.valueOf("table_foo"),
465       HConstants.EMPTY_START_ROW, Bytes.toBytes("a"), false, regionId, 0);
466     HRegionInfo merged = new HRegionInfo(TableName.valueOf("table_foo"),
467       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId+1, 0);
468 
469     Table meta = MetaTableAccessor.getMetaHTable(connection);
470     try {
471       List<HRegionInfo> regionInfos = Lists.newArrayList(parentA, parentB);
472       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
473 
474       MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3,
475           HConstants.LATEST_TIMESTAMP);
476 
477       assertEmptyMetaLocation(meta, merged.getRegionName(), 1);
478       assertEmptyMetaLocation(meta, merged.getRegionName(), 2);
479     } finally {
480       meta.close();
481     }
482   }
483 
484   /**
485    * Tests whether maximum of masters system time versus RSs local system time is used
486    */
487   @Test
488   public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException {
489     long regionId = System.currentTimeMillis();
490     HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf("table_foo"),
491       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
492 
493     ServerName sn = ServerName.valueOf("bar", 0, 0);
494     Table meta = MetaTableAccessor.getMetaHTable(connection);
495     try {
496       List<HRegionInfo> regionInfos = Lists.newArrayList(regionInfo);
497       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
498 
499       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
500       MetaTableAccessor.updateRegionLocation(connection, regionInfo, sn, 1, masterSystemTime);
501 
502       Get get = new Get(regionInfo.getRegionName());
503       Result result = meta.get(get);
504       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
505           MetaTableAccessor.getServerColumn(0));
506       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
507         MetaTableAccessor.getStartCodeColumn(0));
508       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
509         MetaTableAccessor.getSeqNumColumn(0));
510       assertNotNull(serverCell);
511       assertNotNull(startCodeCell);
512       assertNotNull(seqNumCell);
513       assertTrue(serverCell.getValueLength() > 0);
514       assertTrue(startCodeCell.getValueLength() > 0);
515       assertTrue(seqNumCell.getValueLength() > 0);
516       assertEquals(masterSystemTime, serverCell.getTimestamp());
517       assertEquals(masterSystemTime, startCodeCell.getTimestamp());
518       assertEquals(masterSystemTime, seqNumCell.getTimestamp());
519     } finally {
520       meta.close();
521     }
522   }
523 
524   @Test
525   public void testMastersSystemTimeIsUsedInMergeRegions() throws IOException {
526     long regionId = System.currentTimeMillis();
527     HRegionInfo regionInfoA = new HRegionInfo(TableName.valueOf("table_foo"),
528       HConstants.EMPTY_START_ROW, new byte[] {'a'}, false, regionId, 0);
529     HRegionInfo regionInfoB = new HRegionInfo(TableName.valueOf("table_foo"),
530       new byte[] {'a'}, HConstants.EMPTY_END_ROW, false, regionId, 0);
531     HRegionInfo mergedRegionInfo = new HRegionInfo(TableName.valueOf("table_foo"),
532       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0);
533 
534     ServerName sn = ServerName.valueOf("bar", 0, 0);
535     Table meta = MetaTableAccessor.getMetaHTable(connection);
536     try {
537       List<HRegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB);
538       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
539 
540       // write the serverName column with a big current time, but set the masters time as even
541       // bigger. When region merge deletes the rows for regionA and regionB, the serverName columns
542       // should not be seen by the following get
543       long serverNameTime = EnvironmentEdgeManager.currentTime()   + 100000000;
544       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
545 
546       // write the serverName columns
547       MetaTableAccessor.updateRegionLocation(connection, regionInfoA, sn, 1, serverNameTime);
548 
549       // assert that we have the serverName column with expected ts
550       Get get = new Get(mergedRegionInfo.getRegionName());
551       Result result = meta.get(get);
552       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
553           MetaTableAccessor.getServerColumn(0));
554       assertNotNull(serverCell);
555       assertEquals(serverNameTime, serverCell.getTimestamp());
556 
557       // now merge the regions, effectively deleting the rows for region a and b.
558       MetaTableAccessor.mergeRegions(connection, mergedRegionInfo,
559         regionInfoA, regionInfoB, sn, 1, masterSystemTime);
560 
561       result = meta.get(get);
562       serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
563           MetaTableAccessor.getServerColumn(0));
564       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
565         MetaTableAccessor.getStartCodeColumn(0));
566       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
567         MetaTableAccessor.getSeqNumColumn(0));
568       assertNull(serverCell);
569       assertNull(startCodeCell);
570       assertNull(seqNumCell);
571     } finally {
572       meta.close();
573     }
574   }
575 
576   public static class SpyingRpcSchedulerFactory extends SimpleRpcSchedulerFactory {
577     @Override
578     public RpcScheduler create(Configuration conf, PriorityFunction priority, Abortable server) {
579       final RpcScheduler delegate = super.create(conf, priority, server);
580       return new SpyingRpcScheduler(delegate);
581     }
582   }
583 
584   public static class SpyingRpcScheduler extends DelegatingRpcScheduler {
585     long numPriorityCalls = 0;
586 
587     public SpyingRpcScheduler(RpcScheduler delegate) {
588       super(delegate);
589     }
590 
591     @Override
592     public boolean dispatch(CallRunner task) throws IOException, InterruptedException {
593       int priority = task.getCall().getPriority();
594 
595       if (priority > HConstants.QOS_THRESHOLD) {
596         numPriorityCalls++;
597       }
598       return super.dispatch(task);
599     }
600   }
601 
602   @Test
603   public void testMetaUpdatesGoToPriorityQueue() throws Exception {
604     // This test has to be end-to-end, and do the verification from the server side
605     Configuration c = UTIL.getConfiguration();
606 
607     c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
608       SpyingRpcSchedulerFactory.class.getName());
609 
610     // restart so that new config takes place
611     afterClass();
612     beforeClass();
613 
614     TableName tableName = TableName.valueOf("foo");
615     try (Admin admin = connection.getAdmin();
616         RegionLocator rl = connection.getRegionLocator(tableName)) {
617 
618       // create a table and prepare for a manual split
619       UTIL.createTable(tableName, "cf1");
620 
621       HRegionLocation loc = rl.getAllRegionLocations().get(0);
622       HRegionInfo parent = loc.getRegionInfo();
623       long rid = 1000;
624       byte[] splitKey = Bytes.toBytes("a");
625       HRegionInfo splitA = new HRegionInfo(parent.getTable(), parent.getStartKey(),
626         splitKey, false, rid);
627       HRegionInfo splitB = new HRegionInfo(parent.getTable(), splitKey,
628         parent.getEndKey(), false, rid);
629 
630       // find the meta server
631       MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster();
632       int rsIndex = cluster.getServerWithMeta();
633       HRegionServer rs;
634       if (rsIndex >= 0) {
635         rs = cluster.getRegionServer(rsIndex);
636       } else {
637         // it is in master
638         rs = cluster.getMaster();
639       }
640       SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler();
641       long prevCalls = scheduler.numPriorityCalls;
642       MetaTableAccessor.splitRegion(connection, parent, splitA, splitB, loc.getServerName(), 1);
643 
644       assertTrue(prevCalls < scheduler.numPriorityCalls);
645     }
646   }
647 }
648