View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.net.InetSocketAddress;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Random;
35  import java.util.concurrent.atomic.AtomicInteger;
36  
37  import org.apache.commons.logging.Log;
38  import org.apache.commons.logging.LogFactory;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.NamespaceDescriptor;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.client.Admin;
51  import org.apache.hadoop.hbase.client.Connection;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.client.MetaScanner;
54  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
55  import org.apache.hadoop.hbase.client.Result;
56  import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
57  import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
58  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
59  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
60  import org.apache.hadoop.hbase.regionserver.HRegionServer;
61  import org.apache.hadoop.hbase.regionserver.Region;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.hbase.util.Pair;
64  import org.apache.zookeeper.KeeperException;
65  import org.junit.AfterClass;
66  import org.junit.BeforeClass;
67  import org.junit.Ignore;
68  import org.junit.Test;
69  import org.junit.experimental.categories.Category;
70  
71  
72  @Category(MediumTests.class)
73  public class TestRegionPlacement {
74    private static final Log LOG = LogFactory.getLog(TestRegionPlacement.class);
75    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
76    private final static int SLAVES = 10;
77    private static Connection CONNECTION;
78    private static Admin admin;
79    private static RegionPlacementMaintainer rp;
80    private static Position[] positions = Position.values();
81    private int lastRegionOnPrimaryRSCount = 0;
82    private int REGION_NUM = 10;
83    private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
84        new HashMap<HRegionInfo, ServerName[]>();
85  
86    @BeforeClass
87    public static void setupBeforeClass() throws Exception {
88      Configuration conf = TEST_UTIL.getConfiguration();
89      // Enable the favored nodes based load balancer
90      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
91          FavoredNodeLoadBalancer.class, LoadBalancer.class);
92      conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
93      TEST_UTIL.startMiniCluster(SLAVES);
94      CONNECTION = TEST_UTIL.getConnection();
95      admin = CONNECTION.getAdmin();
96      rp = new RegionPlacementMaintainer(conf);
97    }
98  
99    @AfterClass
100   public static void tearDownAfterClass() throws Exception {
101     TEST_UTIL.shutdownMiniCluster();
102   }
103 
104   @Ignore ("Test for unfinished feature") @Test
105   public void testRegionPlacement() throws Exception {
106     String tableStr = "testRegionAssignment";
107     TableName table = TableName.valueOf(tableStr);
108     // Create a table with REGION_NUM regions.
109     createTable(table, REGION_NUM);
110 
111     TEST_UTIL.waitTableAvailable(table);
112 
113     // Verify all the user regions are assigned to the primary region server
114     // based on the plan
115     verifyRegionOnPrimaryRS(REGION_NUM);
116 
117     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
118     // Verify all the region server are update with the latest favored nodes
119     verifyRegionServerUpdated(currentPlan);
120     // Test Case 2: To verify whether the region placement tools can
121     // correctly update the new assignment plan to hbase:meta and Region Server.
122     // The new assignment plan is generated by shuffle the existing assignment
123     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
124     // Shuffle the plan by switching the secondary region server with
125     // the tertiary.
126 
127     // Shuffle the secondary with tertiary favored nodes
128     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
129         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
130     // Let the region placement update the hbase:meta and Region Servers
131     rp.updateAssignmentPlan(shuffledPlan);
132 
133     // Verify the region assignment. There are supposed to no region reassignment
134     // All the regions are still on the primary region server
135     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
136 
137     // Shuffle the plan by switching the primary with secondary and
138     // verify the region reassignment is consistent with the plan.
139     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
140         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
141 
142     // Let the region placement update the hbase:meta and Region Servers
143     rp.updateAssignmentPlan(shuffledPlan);
144 
145     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
146 
147     // also verify that the AssignmentVerificationReport has the correct information
148     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
149     // we are interested in only one table (and hence one report)
150     rp.setTargetTableName(new String[]{tableStr});
151     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
152     AssignmentVerificationReport report = reports.get(0);
153     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
154     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
155     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
156     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
157     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
158     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
159     assertTrue(report.getUnassignedRegions().size() == 0);
160 
161     // Check when a RS stops, the regions get assigned to their secondary/tertiary
162     killRandomServerAndVerifyAssignment();
163     
164     // also verify that the AssignmentVerificationReport has the correct information
165     reports = rp.verifyRegionPlacement(false);
166     report = reports.get(0);
167     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
168     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
169     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
170     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
171     assertTrue("secondary " +
172     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
173         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
174         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
175         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
176     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
177         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
178         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
179     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
180   }
181 
182   private void killRandomServerAndVerifyAssignment() 
183       throws IOException, InterruptedException, KeeperException {
184     ServerName serverToKill = null;
185     int killIndex = 0;
186     Random random = new Random(System.currentTimeMillis());
187     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
188     LOG.debug("Server holding meta " + metaServer);
189     boolean isNamespaceServer = false;
190     do {
191       // kill a random non-meta server carrying at least one region
192       killIndex = random.nextInt(SLAVES);
193       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
194       Collection<Region> regs =
195           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
196       isNamespaceServer = false;
197       for (Region r : regs) {
198         if (r.getRegionInfo().getTable().getNamespaceAsString()
199             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
200           isNamespaceServer = true;
201           break;
202         }
203       }
204     } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
205         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
206     LOG.debug("Stopping RS " + serverToKill);
207     Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
208         new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
209     // mark the regions to track
210     for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
211       ServerName s = entry.getValue()[0];
212       if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
213         regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
214             entry.getValue()[1], entry.getValue()[2]));
215         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
216             entry.getValue()[1] + " " + entry.getValue()[2]);
217       }
218     }
219     int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
220     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
221     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
222     int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
223     while (curr - orig < regionsToVerify.size()) {
224       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
225           " Current #regions " + curr + " Original #regions " + orig);
226       Thread.sleep(200);
227       curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
228     }
229     // now verify
230     for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
231       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
232           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
233       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
234       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
235           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
236           + "/" + secondaryTertiaryServers.getSecond());
237       if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
238           ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
239         fail("Region " + entry.getKey() + " not present on any of the expected servers");
240       }
241     }
242     // start(reinstate) region server since we killed one before
243     TEST_UTIL.getHBaseCluster().startRegionServer();
244   }
245 
246   /**
247    * Used to test the correctness of this class.
248    */
249   @Ignore ("Test for unfinished feature") @Test
250   public void testRandomizedMatrix() {
251     int rows = 100;
252     int cols = 100;
253     float[][] matrix = new float[rows][cols];
254     Random random = new Random();
255     for (int i = 0; i < rows; i++) {
256       for (int j = 0; j < cols; j++) {
257         matrix[i][j] = random.nextFloat();
258       }
259     }
260 
261     // Test that inverting a transformed matrix gives the original matrix.
262     RegionPlacementMaintainer.RandomizedMatrix rm =
263       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
264     float[][] transformed = rm.transform(matrix);
265     float[][] invertedTransformed = rm.invert(transformed);
266     for (int i = 0; i < rows; i++) {
267       for (int j = 0; j < cols; j++) {
268         if (matrix[i][j] != invertedTransformed[i][j]) {
269           throw new RuntimeException();
270         }
271       }
272     }
273 
274     // Test that the indices on a transformed matrix can be inverted to give
275     // the same values on the original matrix.
276     int[] transformedIndices = new int[rows];
277     for (int i = 0; i < rows; i++) {
278       transformedIndices[i] = random.nextInt(cols);
279     }
280     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
281     float[] transformedValues = new float[rows];
282     float[] invertedTransformedValues = new float[rows];
283     for (int i = 0; i < rows; i++) {
284       transformedValues[i] = transformed[i][transformedIndices[i]];
285       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
286     }
287     Arrays.sort(transformedValues);
288     Arrays.sort(invertedTransformedValues);
289     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
290       throw new RuntimeException();
291     }
292   }
293 
294   /**
295    * Shuffle the assignment plan by switching two favored node positions.
296    * @param plan The assignment plan
297    * @param p1 The first switch position
298    * @param p2 The second switch position
299    * @return
300    */
301   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
302       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
303     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
304 
305     for (Map.Entry<HRegionInfo, List<ServerName>> entry :
306       plan.getAssignmentMap().entrySet()) {
307       HRegionInfo region = entry.getKey();
308 
309       // copy the server list from the original plan
310       List<ServerName> shuffledServerList = new ArrayList<ServerName>();
311       shuffledServerList.addAll(entry.getValue());
312 
313       // start to shuffle
314       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
315       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
316 
317       // update the plan
318       shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
319     }
320     return shuffledPlan;
321   }
322 
323   /**
324    * To verify the region assignment status.
325    * It will check the assignment plan consistency between hbase:meta and
326    * region servers.
327    * Also it will verify weather the number of region movement and
328    * the number regions on the primary region server are expected
329    *
330    * @param plan
331    * @param regionMovementNum
332    * @param numRegionsOnPrimaryRS
333    * @throws InterruptedException
334    * @throws IOException
335    */
336   private void verifyRegionAssignment(FavoredNodesPlan plan,
337       int regionMovementNum, int numRegionsOnPrimaryRS)
338   throws InterruptedException, IOException {
339     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
340     verifyMETAUpdated(plan);
341 
342     // Verify the number of region movement is expected
343     verifyRegionMovementNum(regionMovementNum);
344 
345     // Verify the number of regions is assigned to the primary region server
346     // based on the plan is expected
347     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
348 
349     // Verify all the online region server are updated with the assignment plan
350     verifyRegionServerUpdated(plan);
351   }
352 
353   /**
354    * Verify the meta has updated to the latest assignment plan
355    * @param plan
356    * @throws IOException
357    */
358   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
359   throws IOException {
360     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
361     assertTrue("The assignment plan is NOT consistent with the expected plan ",
362         planFromMETA.equals(expectedPlan));
363   }
364 
365   /**
366    * Verify the number of region movement is expected
367    */
368   private void verifyRegionMovementNum(int expected)
369       throws InterruptedException, IOException {
370     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
371     HMaster m = cluster.getMaster();
372     int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
373     // get the assignments start to execute
374     m.balance();
375 
376     int retry = 10;
377     long sleep = 3000;
378     int attempt = 0;
379     int currentRegionOpened, regionMovement;
380     do {
381       currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
382       regionMovement= currentRegionOpened - lastRegionOpenedCount;
383       LOG.debug("There are " + regionMovement + "/" + expected +
384           " regions moved after " + attempt + " attempts");
385       Thread.sleep((++attempt) * sleep);
386     } while (regionMovement != expected && attempt <= retry);
387 
388     // update the lastRegionOpenedCount
389     lastRegionOpenedCount = currentRegionOpened;
390 
391     assertEquals("There are only " + regionMovement + " instead of "
392           + expected + " region movement for " + attempt + " attempts",
393           regionMovement, expected);
394   }
395 
396   /**
397    * Verify the number of user regions is assigned to the primary
398    * region server based on the plan is expected
399    * @param expectedNum.
400    * @throws IOException
401    */
402   private void verifyRegionOnPrimaryRS(int expectedNum)
403       throws IOException {
404     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
405     assertEquals("Only " +  expectedNum + " of user regions running " +
406         "on the primary region server", expectedNum ,
407         lastRegionOnPrimaryRSCount);
408   }
409 
410   /**
411    * Verify all the online region servers has been updated to the
412    * latest assignment plan
413    * @param plan
414    * @throws IOException
415    */
416   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
417     // Verify all region servers contain the correct favored nodes information
418     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
419     for (int i = 0; i < SLAVES; i++) {
420       HRegionServer rs = cluster.getRegionServer(i);
421       for (Region region: rs.getOnlineRegions(TableName.valueOf("testRegionAssignment"))) {
422         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
423             region.getRegionInfo().getEncodedName());
424         List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
425 
426         // All regions are supposed to have favored nodes,
427         // except for hbase:meta and ROOT
428         if (favoredServerList == null) {
429           HTableDescriptor desc = region.getTableDesc();
430           // Verify they are ROOT and hbase:meta regions since no favored nodes
431           assertNull(favoredSocketAddress);
432           assertTrue("User region " +
433               region.getTableDesc().getTableName() +
434               " should have favored nodes",
435               (desc.isRootRegion() || desc.isMetaRegion()));
436         } else {
437           // For user region, the favored nodes in the region server should be
438           // identical to favored nodes in the assignmentPlan
439           assertTrue(favoredSocketAddress.length == favoredServerList.size());
440           assertTrue(favoredServerList.size() > 0);
441           for (int j = 0; j < favoredServerList.size(); j++) {
442             InetSocketAddress addrFromRS = favoredSocketAddress[j];
443             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
444                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
445 
446             assertNotNull(addrFromRS);
447             assertNotNull(addrFromPlan);
448             assertTrue("Region server " + rs.getServerName().getHostAndPort()
449                 + " has the " + positions[j] +
450                 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
451                 addrFromRS + " which is inconsistent with the plan "
452                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
453           }
454         }
455       }
456     }
457   }
458 
459   /**
460    * Check whether regions are assigned to servers consistent with the explicit
461    * hints that are persisted in the hbase:meta table.
462    * Also keep track of the number of the regions are assigned to the
463    * primary region server.
464    * @return the number of regions are assigned to the primary region server
465    * @throws IOException
466    */
467   private int getNumRegionisOnPrimaryRS() throws IOException {
468     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
469     final AtomicInteger totalRegionNum = new AtomicInteger(0);
470     LOG.info("The start of region placement verification");
471     MetaScannerVisitor visitor = new MetaScannerVisitor() {
472       public boolean processRow(Result result) throws IOException {
473         try {
474           @SuppressWarnings("deprecation")
475           HRegionInfo info = MetaScanner.getHRegionInfo(result);
476           if(info.getTable().getNamespaceAsString()
477               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
478             return true;
479           }
480           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
481               HConstants.SERVER_QUALIFIER);
482           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
483               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
484           // Add the favored nodes into assignment plan
485           ServerName[] favoredServerList =
486               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
487           favoredNodesAssignmentPlan.put(info, favoredServerList);
488 
489           Position[] positions = Position.values();
490           if (info != null) {
491             totalRegionNum.incrementAndGet();
492             if (server != null) {
493               ServerName serverName =
494                   ServerName.valueOf(Bytes.toString(server), -1);
495               if (favoredNodes != null) {
496                 String placement = "[NOT FAVORED NODE]";
497                 for (int i = 0; i < favoredServerList.length; i++) {
498                   if (favoredServerList[i].equals(serverName)) {
499                     placement = positions[i].toString();
500                     if (i == Position.PRIMARY.ordinal()) {
501                       regionOnPrimaryNum.incrementAndGet();
502                     }
503                     break;
504                   }
505                 }
506                 LOG.info(info.getRegionNameAsString() + " on " +
507                     serverName + " " + placement);
508               } else {
509                 LOG.info(info.getRegionNameAsString() + " running on " +
510                     serverName + " but there is no favored region server");
511               }
512             } else {
513               LOG.info(info.getRegionNameAsString() +
514                   " not assigned to any server");
515             }
516           }
517           return true;
518         } catch (RuntimeException e) {
519           LOG.error("Result=" + result);
520           throw e;
521         }
522       }
523 
524       @Override
525       public void close() throws IOException {}
526     };
527     MetaScanner.metaScan(CONNECTION, visitor);
528     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
529         totalRegionNum.intValue() + " regions running on the primary" +
530         " region servers" );
531     return regionOnPrimaryNum.intValue() ;
532   }
533 
534   /**
535    * Create a table with specified table name and region number.
536    * @param tablename
537    * @param regionNum
538    * @return
539    * @throws IOException
540    */
541   private static void createTable(TableName tableName, int regionNum)
542       throws IOException {
543     int expectedRegions = regionNum;
544     byte[][] splitKeys = new byte[expectedRegions - 1][];
545     for (int i = 1; i < expectedRegions; i++) {
546       byte splitKey = (byte) i;
547       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
548     }
549 
550     HTableDescriptor desc = new HTableDescriptor(tableName);
551     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
552     admin.createTable(desc, splitKeys);
553 
554     HTable ht = (HTable) CONNECTION.getTable(tableName);
555     @SuppressWarnings("deprecation")
556     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
557     assertEquals("Tried to create " + expectedRegions + " regions "
558         + "but only found " + regions.size(), expectedRegions, regions.size());
559     ht.close();
560   }
561 }