1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.net.InetSocketAddress;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Random;
35 import java.util.concurrent.atomic.AtomicInteger;
36
37 import org.apache.commons.logging.Log;
38 import org.apache.commons.logging.LogFactory;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.testclassification.MediumTests;
46 import org.apache.hadoop.hbase.MiniHBaseCluster;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.client.Admin;
51 import org.apache.hadoop.hbase.client.Connection;
52 import org.apache.hadoop.hbase.client.HTable;
53 import org.apache.hadoop.hbase.client.MetaScanner;
54 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
55 import org.apache.hadoop.hbase.client.Result;
56 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
57 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
58 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
59 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
60 import org.apache.hadoop.hbase.regionserver.HRegionServer;
61 import org.apache.hadoop.hbase.regionserver.Region;
62 import org.apache.hadoop.hbase.util.Bytes;
63 import org.apache.hadoop.hbase.util.Pair;
64 import org.apache.zookeeper.KeeperException;
65 import org.junit.AfterClass;
66 import org.junit.BeforeClass;
67 import org.junit.Ignore;
68 import org.junit.Test;
69 import org.junit.experimental.categories.Category;
70
71
72 @Category(MediumTests.class)
73 public class TestRegionPlacement {
74 private static final Log LOG = LogFactory.getLog(TestRegionPlacement.class);
75 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
76 private final static int SLAVES = 10;
77 private static Connection CONNECTION;
78 private static Admin admin;
79 private static RegionPlacementMaintainer rp;
80 private static Position[] positions = Position.values();
81 private int lastRegionOnPrimaryRSCount = 0;
82 private int REGION_NUM = 10;
83 private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
84 new HashMap<HRegionInfo, ServerName[]>();
85
86 @BeforeClass
87 public static void setupBeforeClass() throws Exception {
88 Configuration conf = TEST_UTIL.getConfiguration();
89
90 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
91 FavoredNodeLoadBalancer.class, LoadBalancer.class);
92 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
93 TEST_UTIL.startMiniCluster(SLAVES);
94 CONNECTION = TEST_UTIL.getConnection();
95 admin = CONNECTION.getAdmin();
96 rp = new RegionPlacementMaintainer(conf);
97 }
98
99 @AfterClass
100 public static void tearDownAfterClass() throws Exception {
101 TEST_UTIL.shutdownMiniCluster();
102 }
103
104 @Ignore ("Test for unfinished feature") @Test
105 public void testRegionPlacement() throws Exception {
106 String tableStr = "testRegionAssignment";
107 TableName table = TableName.valueOf(tableStr);
108
109 createTable(table, REGION_NUM);
110
111 TEST_UTIL.waitTableAvailable(table);
112
113
114
115 verifyRegionOnPrimaryRS(REGION_NUM);
116
117 FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
118
119 verifyRegionServerUpdated(currentPlan);
120
121
122
123
124
125
126
127
128 FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
129 FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
130
131 rp.updateAssignmentPlan(shuffledPlan);
132
133
134
135 verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
136
137
138
139 shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
140 FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
141
142
143 rp.updateAssignmentPlan(shuffledPlan);
144
145 verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
146
147
148 RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
149
150 rp.setTargetTableName(new String[]{tableStr});
151 List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
152 AssignmentVerificationReport report = reports.get(0);
153 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
154 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
155 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
156 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
157 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
158 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
159 assertTrue(report.getUnassignedRegions().size() == 0);
160
161
162 killRandomServerAndVerifyAssignment();
163
164
165 reports = rp.verifyRegionPlacement(false);
166 report = reports.get(0);
167 assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
168 assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
169 assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
170 assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
171 assertTrue("secondary " +
172 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
173 + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
174 (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
175 || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
176 assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
177 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
178 report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
179 RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
180 }
181
182 private void killRandomServerAndVerifyAssignment()
183 throws IOException, InterruptedException, KeeperException {
184 ServerName serverToKill = null;
185 int killIndex = 0;
186 Random random = new Random(System.currentTimeMillis());
187 ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
188 LOG.debug("Server holding meta " + metaServer);
189 boolean isNamespaceServer = false;
190 do {
191
192 killIndex = random.nextInt(SLAVES);
193 serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
194 Collection<Region> regs =
195 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
196 isNamespaceServer = false;
197 for (Region r : regs) {
198 if (r.getRegionInfo().getTable().getNamespaceAsString()
199 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
200 isNamespaceServer = true;
201 break;
202 }
203 }
204 } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
205 TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
206 LOG.debug("Stopping RS " + serverToKill);
207 Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
208 new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
209
210 for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
211 ServerName s = entry.getValue()[0];
212 if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
213 regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
214 entry.getValue()[1], entry.getValue()[2]));
215 LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
216 entry.getValue()[1] + " " + entry.getValue()[2]);
217 }
218 }
219 int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
220 TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
221 TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
222 int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
223 while (curr - orig < regionsToVerify.size()) {
224 LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
225 " Current #regions " + curr + " Original #regions " + orig);
226 Thread.sleep(200);
227 curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
228 }
229
230 for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
231 ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
232 .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
233 Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
234 LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
235 " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
236 + "/" + secondaryTertiaryServers.getSecond());
237 if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
238 ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
239 fail("Region " + entry.getKey() + " not present on any of the expected servers");
240 }
241 }
242
243 TEST_UTIL.getHBaseCluster().startRegionServer();
244 }
245
246
247
248
249 @Ignore ("Test for unfinished feature") @Test
250 public void testRandomizedMatrix() {
251 int rows = 100;
252 int cols = 100;
253 float[][] matrix = new float[rows][cols];
254 Random random = new Random();
255 for (int i = 0; i < rows; i++) {
256 for (int j = 0; j < cols; j++) {
257 matrix[i][j] = random.nextFloat();
258 }
259 }
260
261
262 RegionPlacementMaintainer.RandomizedMatrix rm =
263 new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
264 float[][] transformed = rm.transform(matrix);
265 float[][] invertedTransformed = rm.invert(transformed);
266 for (int i = 0; i < rows; i++) {
267 for (int j = 0; j < cols; j++) {
268 if (matrix[i][j] != invertedTransformed[i][j]) {
269 throw new RuntimeException();
270 }
271 }
272 }
273
274
275
276 int[] transformedIndices = new int[rows];
277 for (int i = 0; i < rows; i++) {
278 transformedIndices[i] = random.nextInt(cols);
279 }
280 int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
281 float[] transformedValues = new float[rows];
282 float[] invertedTransformedValues = new float[rows];
283 for (int i = 0; i < rows; i++) {
284 transformedValues[i] = transformed[i][transformedIndices[i]];
285 invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
286 }
287 Arrays.sort(transformedValues);
288 Arrays.sort(invertedTransformedValues);
289 if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
290 throw new RuntimeException();
291 }
292 }
293
294
295
296
297
298
299
300
301 private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
302 FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
303 FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
304
305 for (Map.Entry<HRegionInfo, List<ServerName>> entry :
306 plan.getAssignmentMap().entrySet()) {
307 HRegionInfo region = entry.getKey();
308
309
310 List<ServerName> shuffledServerList = new ArrayList<ServerName>();
311 shuffledServerList.addAll(entry.getValue());
312
313
314 shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
315 shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
316
317
318 shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
319 }
320 return shuffledPlan;
321 }
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336 private void verifyRegionAssignment(FavoredNodesPlan plan,
337 int regionMovementNum, int numRegionsOnPrimaryRS)
338 throws InterruptedException, IOException {
339
340 verifyMETAUpdated(plan);
341
342
343 verifyRegionMovementNum(regionMovementNum);
344
345
346
347 verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
348
349
350 verifyRegionServerUpdated(plan);
351 }
352
353
354
355
356
357
358 private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
359 throws IOException {
360 FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
361 assertTrue("The assignment plan is NOT consistent with the expected plan ",
362 planFromMETA.equals(expectedPlan));
363 }
364
365
366
367
368 private void verifyRegionMovementNum(int expected)
369 throws InterruptedException, IOException {
370 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
371 HMaster m = cluster.getMaster();
372 int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
373
374 m.balance();
375
376 int retry = 10;
377 long sleep = 3000;
378 int attempt = 0;
379 int currentRegionOpened, regionMovement;
380 do {
381 currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
382 regionMovement= currentRegionOpened - lastRegionOpenedCount;
383 LOG.debug("There are " + regionMovement + "/" + expected +
384 " regions moved after " + attempt + " attempts");
385 Thread.sleep((++attempt) * sleep);
386 } while (regionMovement != expected && attempt <= retry);
387
388
389 lastRegionOpenedCount = currentRegionOpened;
390
391 assertEquals("There are only " + regionMovement + " instead of "
392 + expected + " region movement for " + attempt + " attempts",
393 regionMovement, expected);
394 }
395
396
397
398
399
400
401
402 private void verifyRegionOnPrimaryRS(int expectedNum)
403 throws IOException {
404 lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
405 assertEquals("Only " + expectedNum + " of user regions running " +
406 "on the primary region server", expectedNum ,
407 lastRegionOnPrimaryRSCount);
408 }
409
410
411
412
413
414
415
416 private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
417
418 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
419 for (int i = 0; i < SLAVES; i++) {
420 HRegionServer rs = cluster.getRegionServer(i);
421 for (Region region: rs.getOnlineRegions(TableName.valueOf("testRegionAssignment"))) {
422 InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
423 region.getRegionInfo().getEncodedName());
424 List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
425
426
427
428 if (favoredServerList == null) {
429 HTableDescriptor desc = region.getTableDesc();
430
431 assertNull(favoredSocketAddress);
432 assertTrue("User region " +
433 region.getTableDesc().getTableName() +
434 " should have favored nodes",
435 (desc.isRootRegion() || desc.isMetaRegion()));
436 } else {
437
438
439 assertTrue(favoredSocketAddress.length == favoredServerList.size());
440 assertTrue(favoredServerList.size() > 0);
441 for (int j = 0; j < favoredServerList.size(); j++) {
442 InetSocketAddress addrFromRS = favoredSocketAddress[j];
443 InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
444 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
445
446 assertNotNull(addrFromRS);
447 assertNotNull(addrFromPlan);
448 assertTrue("Region server " + rs.getServerName().getHostAndPort()
449 + " has the " + positions[j] +
450 " for region " + region.getRegionInfo().getRegionNameAsString() + " is " +
451 addrFromRS + " which is inconsistent with the plan "
452 + addrFromPlan, addrFromRS.equals(addrFromPlan));
453 }
454 }
455 }
456 }
457 }
458
459
460
461
462
463
464
465
466
467 private int getNumRegionisOnPrimaryRS() throws IOException {
468 final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
469 final AtomicInteger totalRegionNum = new AtomicInteger(0);
470 LOG.info("The start of region placement verification");
471 MetaScannerVisitor visitor = new MetaScannerVisitor() {
472 public boolean processRow(Result result) throws IOException {
473 try {
474 @SuppressWarnings("deprecation")
475 HRegionInfo info = MetaScanner.getHRegionInfo(result);
476 if(info.getTable().getNamespaceAsString()
477 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
478 return true;
479 }
480 byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
481 HConstants.SERVER_QUALIFIER);
482 byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
483 FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
484
485 ServerName[] favoredServerList =
486 FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
487 favoredNodesAssignmentPlan.put(info, favoredServerList);
488
489 Position[] positions = Position.values();
490 if (info != null) {
491 totalRegionNum.incrementAndGet();
492 if (server != null) {
493 ServerName serverName =
494 ServerName.valueOf(Bytes.toString(server), -1);
495 if (favoredNodes != null) {
496 String placement = "[NOT FAVORED NODE]";
497 for (int i = 0; i < favoredServerList.length; i++) {
498 if (favoredServerList[i].equals(serverName)) {
499 placement = positions[i].toString();
500 if (i == Position.PRIMARY.ordinal()) {
501 regionOnPrimaryNum.incrementAndGet();
502 }
503 break;
504 }
505 }
506 LOG.info(info.getRegionNameAsString() + " on " +
507 serverName + " " + placement);
508 } else {
509 LOG.info(info.getRegionNameAsString() + " running on " +
510 serverName + " but there is no favored region server");
511 }
512 } else {
513 LOG.info(info.getRegionNameAsString() +
514 " not assigned to any server");
515 }
516 }
517 return true;
518 } catch (RuntimeException e) {
519 LOG.error("Result=" + result);
520 throw e;
521 }
522 }
523
524 @Override
525 public void close() throws IOException {}
526 };
527 MetaScanner.metaScan(CONNECTION, visitor);
528 LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
529 totalRegionNum.intValue() + " regions running on the primary" +
530 " region servers" );
531 return regionOnPrimaryNum.intValue() ;
532 }
533
534
535
536
537
538
539
540
541 private static void createTable(TableName tableName, int regionNum)
542 throws IOException {
543 int expectedRegions = regionNum;
544 byte[][] splitKeys = new byte[expectedRegions - 1][];
545 for (int i = 1; i < expectedRegions; i++) {
546 byte splitKey = (byte) i;
547 splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
548 }
549
550 HTableDescriptor desc = new HTableDescriptor(tableName);
551 desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
552 admin.createTable(desc, splitKeys);
553
554 HTable ht = (HTable) CONNECTION.getTable(tableName);
555 @SuppressWarnings("deprecation")
556 Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
557 assertEquals("Tried to create " + expectedRegions + " regions "
558 + "but only found " + regions.size(), expectedRegions, regions.size());
559 ht.close();
560 }
561 }