1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.InvocationTargetException;
30 import java.lang.reflect.Method;
31 import java.util.List;
32 import java.util.Map;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.client.Admin;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.HConnection;
40 import org.apache.hadoop.hbase.client.HConnectionManager;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.ResultScanner;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.client.Table;
47 import org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination;
48 import org.apache.hadoop.hbase.master.HMaster;
49 import org.apache.hadoop.hbase.master.LoadBalancer;
50 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
51 import org.apache.hadoop.hbase.testclassification.LargeTests;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.Threads;
55 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
56 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
57 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
58 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
59 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
60 import org.apache.zookeeper.CreateMode;
61 import org.apache.zookeeper.KeeperException;
62 import org.apache.zookeeper.ZooDefs;
63 import org.apache.zookeeper.ZooKeeper;
64 import org.apache.zookeeper.ZooKeeper.States;
65 import org.apache.zookeeper.data.ACL;
66 import org.apache.zookeeper.data.Stat;
67 import org.junit.After;
68 import org.junit.AfterClass;
69 import org.junit.Assert;
70 import org.junit.Before;
71 import org.junit.BeforeClass;
72 import org.junit.Test;
73 import org.junit.experimental.categories.Category;
74
75
76
77 @Category(LargeTests.class)
78 public class TestZooKeeper {
79 private static final Log LOG = LogFactory.getLog(TestZooKeeper.class);
80
81 private final static HBaseTestingUtility
82 TEST_UTIL = new HBaseTestingUtility();
83
84
85
86
87 @BeforeClass
88 public static void setUpBeforeClass() throws Exception {
89
90 Configuration conf = TEST_UTIL.getConfiguration();
91 TEST_UTIL.startMiniDFSCluster(2);
92 TEST_UTIL.startMiniZKCluster();
93 conf.setBoolean("dfs.support.append", true);
94 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
95 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
96 LoadBalancer.class);
97 }
98
99
100
101
102 @AfterClass
103 public static void tearDownAfterClass() throws Exception {
104 TEST_UTIL.shutdownMiniCluster();
105 }
106
107
108
109
110 @Before
111 public void setUp() throws Exception {
112 TEST_UTIL.startMiniHBaseCluster(2, 2);
113 }
114
115 @After
116 public void after() throws Exception {
117 try {
118
119
120 TEST_UTIL.getHBaseCluster().killAll();
121
122
123 TEST_UTIL.shutdownMiniHBaseCluster();
124 } finally {
125 TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
126 ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
127 }
128 }
129
130 private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
131 throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
132 Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
133 getterZK.setAccessible(true);
134 return (ZooKeeperWatcher) getterZK.invoke(c);
135 }
136
137
138
139
140
141
142
143
144
145 public void testClientSessionExpired() throws Exception {
146 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
147
148
149 c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
150
151 HConnection connection = HConnectionManager.getConnection(c);
152
153 ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
154 LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
155 connectionZK.hashCode()));
156 LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
157 connectionZK.getRecoverableZooKeeper().hashCode()));
158 LOG.info("session="+Long.toHexString(
159 connectionZK.getRecoverableZooKeeper().getSessionId()));
160
161 TEST_UTIL.expireSession(connectionZK);
162
163 LOG.info("Before using zkw state=" +
164 connectionZK.getRecoverableZooKeeper().getState());
165
166 try {
167 connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
168 "/1/1", false);
169 } catch (KeeperException ignored) {
170 }
171
172
173 States state = connectionZK.getRecoverableZooKeeper().getState();
174 LOG.info("After using zkw state=" + state);
175 LOG.info("session="+Long.toHexString(
176 connectionZK.getRecoverableZooKeeper().getSessionId()));
177
178
179 final long limit1 = System.currentTimeMillis() + 3000;
180 while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
181 state = connectionZK.getRecoverableZooKeeper().getState();
182 }
183 LOG.info("After using zkw loop=" + state);
184 LOG.info("ZooKeeper should have timed out");
185 LOG.info("session="+Long.toHexString(
186 connectionZK.getRecoverableZooKeeper().getSessionId()));
187
188
189
190
191
192
193
194 ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
195
196 States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
197 LOG.info("After new get state=" +state2);
198
199
200
201 final long limit2 = System.currentTimeMillis() + 3000;
202 while (System.currentTimeMillis() < limit2 &&
203 state2 != States.CONNECTED && state2 != States.CONNECTING) {
204
205 newConnectionZK = getZooKeeperWatcher(connection);
206 state2 = newConnectionZK.getRecoverableZooKeeper().getState();
207 }
208 LOG.info("After new get state loop=" + state2);
209
210 Assert.assertTrue(
211 state2 == States.CONNECTED || state2 == States.CONNECTING);
212
213 connection.close();
214 }
215
216 @Test (timeout = 120000)
217 public void testRegionServerSessionExpired() throws Exception {
218 LOG.info("Starting testRegionServerSessionExpired");
219 TEST_UTIL.expireRegionServerSession(0);
220 testSanity("testRegionServerSessionExpired");
221 }
222
223 @Test(timeout = 300000)
224 public void testMasterSessionExpired() throws Exception {
225 LOG.info("Starting testMasterSessionExpired");
226 TEST_UTIL.expireMasterSession();
227 testSanity("testMasterSessionExpired");
228 }
229
230
231
232
233
234
235 @Test(timeout = 300000)
236 public void testMasterZKSessionRecoveryFailure() throws Exception {
237 LOG.info("Starting testMasterZKSessionRecoveryFailure");
238 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
239 HMaster m = cluster.getMaster();
240 m.abort("Test recovery from zk session expired",
241 new KeeperException.SessionExpiredException());
242 assertTrue(m.isStopped());
243 testSanity("testMasterZKSessionRecoveryFailure");
244 }
245
246
247
248
249
250 private void testSanity(final String testName) throws Exception{
251 String tableName = testName + "_" + System.currentTimeMillis();
252 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
253 HColumnDescriptor family = new HColumnDescriptor("fam");
254 desc.addFamily(family);
255 LOG.info("Creating table " + tableName);
256 Admin admin = TEST_UTIL.getHBaseAdmin();
257 try {
258 admin.createTable(desc);
259 } finally {
260 admin.close();
261 }
262
263 Table table =
264 new HTable(new Configuration(TEST_UTIL.getConfiguration()), desc.getTableName());
265 Put put = new Put(Bytes.toBytes("testrow"));
266 put.add(Bytes.toBytes("fam"),
267 Bytes.toBytes("col"), Bytes.toBytes("testdata"));
268 LOG.info("Putting table " + tableName);
269 table.put(put);
270 table.close();
271 }
272
273 @Test
274 public void testMultipleZK()
275 throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
276 Table localMeta =
277 new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
278 Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
279 otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
280 Table ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
281
282
283 final byte [] row = new byte [] {'r'};
284 localMeta.exists(new Get(row));
285 ipMeta.exists(new Get(row));
286
287
288 ZooKeeperWatcher z1 =
289 getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
290 ZooKeeperWatcher z2 =
291 getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
292 assertFalse(z1 == z2);
293 assertFalse(z1.getQuorum().equals(z2.getQuorum()));
294
295 localMeta.close();
296 ipMeta.close();
297 }
298
299
300
301
302
303 @Test
304 public void testCreateWithParents() throws Exception {
305 ZooKeeperWatcher zkw =
306 new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
307 TestZooKeeper.class.getName(), null);
308 byte[] expectedData = new byte[] { 1, 2, 3 };
309 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
310 byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
311 assertTrue(Bytes.equals(expectedData, data));
312 ZKUtil.deleteNodeRecursively(zkw, "/l1");
313
314 ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
315 data = ZKUtil.getData(zkw, "/testCreateWithParents");
316 assertTrue(Bytes.equals(expectedData, data));
317 ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
318 }
319
320
321
322
323
324
325 @Test
326 public void testZNodeDeletes() throws Exception {
327 ZooKeeperWatcher zkw = new ZooKeeperWatcher(
328 new Configuration(TEST_UTIL.getConfiguration()),
329 TestZooKeeper.class.getName(), null);
330 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
331 try {
332 ZKUtil.deleteNode(zkw, "/l1/l2");
333 fail("We should not be able to delete if znode has childs");
334 } catch (KeeperException ex) {
335 assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
336 }
337 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
338
339 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
340
341
342 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
343
344 ZKUtil.deleteNode(zkw, "/l1");
345 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
346 }
347
348
349
350
351
352
353
354
355 @Test
356 public void testCreateSilentIsReallySilent() throws InterruptedException,
357 KeeperException, IOException {
358 Configuration c = TEST_UTIL.getConfiguration();
359
360 String aclZnode = "/aclRoot";
361 String quorumServers = ZKConfig.getZKQuorumServersString(c);
362 int sessionTimeout = 5 * 1000;
363 ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
364 zk.addAuthInfo("digest", "hbase:rox".getBytes());
365
366
367
368 ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
369 "testCreateSilentIsReallySilent", null);
370
371
372 Stat s = null;
373 List<ACL> oldACL = null;
374 while (true) {
375 try {
376 s = new Stat();
377 oldACL = zk.getACL("/", s);
378 break;
379 } catch (KeeperException e) {
380 switch (e.code()) {
381 case CONNECTIONLOSS:
382 case SESSIONEXPIRED:
383 case OPERATIONTIMEOUT:
384 LOG.warn("Possibly transient ZooKeeper exception", e);
385 Threads.sleep(100);
386 break;
387 default:
388 throw e;
389 }
390 }
391 }
392
393
394
395 while (true) {
396 try {
397 zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
398 break;
399 } catch (KeeperException e) {
400 switch (e.code()) {
401 case CONNECTIONLOSS:
402 case SESSIONEXPIRED:
403 case OPERATIONTIMEOUT:
404 LOG.warn("Possibly transient ZooKeeper exception: " + e);
405 Threads.sleep(100);
406 break;
407 default:
408 throw e;
409 }
410 }
411 }
412
413 while (true) {
414 try {
415 zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
416 break;
417 } catch (KeeperException e) {
418 switch (e.code()) {
419 case CONNECTIONLOSS:
420 case SESSIONEXPIRED:
421 case OPERATIONTIMEOUT:
422 LOG.warn("Possibly transient ZooKeeper exception: " + e);
423 Threads.sleep(100);
424 break;
425 default:
426 throw e;
427 }
428 }
429 }
430 zk.close();
431 ZKUtil.createAndFailSilent(zk2, aclZnode);
432
433
434 ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
435 zk3.addAuthInfo("digest", "hbase:rox".getBytes());
436 try {
437 zk3.setACL("/", oldACL, -1);
438 } finally {
439 zk3.close();
440 }
441 }
442
443
444
445
446
447 @Test
448 @SuppressWarnings("deprecation")
449 public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
450 throws Exception {
451 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
452 "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
453 ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
454 }
455
456
457
458
459
460
461 @Test(timeout = 300000)
462 public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
463 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
464 cluster.startRegionServer();
465 cluster.waitForActiveAndReadyMaster(10000);
466 HMaster m = cluster.getMaster();
467 final ZooKeeperWatcher zkw = m.getZooKeeper();
468
469 try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
470 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
471 Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
472 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
473 String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
474 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
475 htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
476 admin.createTable(htd, SPLIT_KEYS);
477 TEST_UTIL.waitUntilNoRegionsInTransition(60000);
478 m.getZooKeeper().close();
479 MockLoadBalancer.retainAssignCalled = false;
480 final int expectedNumOfListeners = countPermanentListeners(zkw);
481 m.abort("Test recovery from zk session expired",
482 new KeeperException.SessionExpiredException());
483 assertTrue(m.isStopped());
484
485
486 assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
487
488
489 cluster.waitForActiveAndReadyMaster(120000);
490 final HMaster newMaster = cluster.getMasterThread().getMaster();
491 assertEquals(expectedNumOfListeners, countPermanentListeners(newMaster.getZooKeeper()));
492 }
493 }
494
495
496
497
498
499 private int countPermanentListeners(ZooKeeperWatcher watcher) {
500 return countListeners(watcher, ZkSplitLogWorkerCoordination.class);
501 }
502
503
504
505
506 private int countListeners(ZooKeeperWatcher watcher, Class<?>... exclude) {
507 int cnt = 0;
508 for (Object o : watcher.getListeners()) {
509 boolean skip = false;
510 for (Class<?> aClass : exclude) {
511 if (aClass.isAssignableFrom(o.getClass())) {
512 skip = true;
513 break;
514 }
515 }
516 if (!skip) {
517 cnt += 1;
518 }
519 }
520 return cnt;
521 }
522
523
524
525
526
527
528 @Test(timeout = 300000)
529 public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
530 KeeperException, InterruptedException {
531 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
532 cluster.startRegionServer();
533 HMaster m = cluster.getMaster();
534
535 Admin admin = TEST_UTIL.getHBaseAdmin();
536 Table table = null;
537 try {
538 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
539 Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
540
541 String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
542 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
543 HColumnDescriptor hcd = new HColumnDescriptor("col");
544 htd.addFamily(hcd);
545 admin.createTable(htd, SPLIT_KEYS);
546 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
547 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
548 table = new HTable(TEST_UTIL.getConfiguration(), htd.getTableName());
549 Put p;
550 int numberOfPuts;
551 for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
552 p = new Put(Bytes.toBytes(numberOfPuts));
553 p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
554 table.put(p);
555 }
556 m.getZooKeeper().close();
557 m.abort("Test recovery from zk session expired",
558 new KeeperException.SessionExpiredException());
559 assertTrue(m.isStopped());
560 cluster.getRegionServer(0).abort("Aborting");
561
562
563 Scan scan = new Scan();
564 int numberOfRows = 0;
565 ResultScanner scanner = table.getScanner(scan);
566 Result[] result = scanner.next(1);
567 while (result != null && result.length > 0) {
568 numberOfRows++;
569 result = scanner.next(1);
570 }
571 assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
572 numberOfRows);
573 } finally {
574 if (table != null) table.close();
575 admin.close();
576 }
577 }
578
579 static class MockLoadBalancer extends SimpleLoadBalancer {
580 static boolean retainAssignCalled = false;
581
582 @Override
583 public Map<ServerName, List<HRegionInfo>> retainAssignment(
584 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
585 retainAssignCalled = true;
586 return super.retainAssignment(regions, servers);
587 }
588 }
589
590 }
591