1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.concurrent.Semaphore;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.ChoreService;
32 import org.apache.hadoop.hbase.CoordinatedStateManager;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.testclassification.MediumTests;
35 import org.apache.hadoop.hbase.Server;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.client.ClusterConnection;
38 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
39 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
40 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
41 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
42 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
43 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
44 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
45 import org.apache.zookeeper.KeeperException;
46 import org.junit.AfterClass;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50 import org.mockito.Mockito;
51
52
53
54
55 @Category(MediumTests.class)
56 public class TestActiveMasterManager {
57 private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
58 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59
60 @BeforeClass
61 public static void setUpBeforeClass() throws Exception {
62 TEST_UTIL.startMiniZKCluster();
63 }
64
65 @AfterClass
66 public static void tearDownAfterClass() throws Exception {
67 TEST_UTIL.shutdownMiniZKCluster();
68 }
69
70 @Test public void testRestartMaster() throws IOException, KeeperException {
71 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
72 "testActiveMasterManagerFromZK", null, true);
73 try {
74 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
75 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
76 } catch(KeeperException.NoNodeException nne) {}
77
78
79 ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
80
81 DummyMaster dummyMaster = new DummyMaster(zk,master);
82 ClusterStatusTracker clusterStatusTracker =
83 dummyMaster.getClusterStatusTracker();
84 ActiveMasterManager activeMasterManager =
85 dummyMaster.getActiveMasterManager();
86 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
87
88
89 MonitoredTask status = Mockito.mock(MonitoredTask.class);
90 clusterStatusTracker.setClusterUp();
91
92 activeMasterManager.blockUntilBecomingActiveMaster(100, status);
93 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
94 assertMaster(zk, master);
95
96
97 DummyMaster secondDummyMaster = new DummyMaster(zk,master);
98 ActiveMasterManager secondActiveMasterManager =
99 secondDummyMaster.getActiveMasterManager();
100 assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
101 activeMasterManager.blockUntilBecomingActiveMaster(100, status);
102 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
103 assertMaster(zk, master);
104 }
105
106
107
108
109
110
111 @Test
112 public void testActiveMasterManagerFromZK() throws Exception {
113 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
114 "testActiveMasterManagerFromZK", null, true);
115 try {
116 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
117 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
118 } catch(KeeperException.NoNodeException nne) {}
119
120
121 ServerName firstMasterAddress =
122 ServerName.valueOf("localhost", 1, System.currentTimeMillis());
123 ServerName secondMasterAddress =
124 ServerName.valueOf("localhost", 2, System.currentTimeMillis());
125
126
127 DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
128 ActiveMasterManager activeMasterManager =
129 ms1.getActiveMasterManager();
130 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
131
132
133 ClusterStatusTracker clusterStatusTracker =
134 ms1.getClusterStatusTracker();
135 clusterStatusTracker.setClusterUp();
136 activeMasterManager.blockUntilBecomingActiveMaster(100,
137 Mockito.mock(MonitoredTask.class));
138 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
139 assertMaster(zk, firstMasterAddress);
140
141
142 WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
143 t.start();
144
145
146 int sleeps = 0;
147 while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
148 Thread.sleep(10);
149 sleeps++;
150 }
151
152
153 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
154 assertTrue(t.manager.clusterHasActiveMaster.get());
155
156 assertFalse(t.isActiveMaster);
157
158
159 ms1.stop("stopping first server");
160
161
162 NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
163 zk.registerListener(listener);
164
165 LOG.info("Deleting master node");
166 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
167
168
169 LOG.info("Waiting for active master manager to be notified");
170 listener.waitForDeletion();
171 LOG.info("Master node deleted");
172
173
174
175 sleeps = 0;
176 while(!t.isActiveMaster && sleeps < 100) {
177 Thread.sleep(10);
178 sleeps++;
179 }
180 LOG.debug("Slept " + sleeps + " times");
181
182 assertTrue(t.manager.clusterHasActiveMaster.get());
183 assertTrue(t.isActiveMaster);
184
185 LOG.info("Deleting master node");
186 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
187 }
188
189
190
191
192
193
194
195
196 private void assertMaster(ZooKeeperWatcher zk,
197 ServerName expectedAddress)
198 throws KeeperException, IOException {
199 ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
200 assertNotNull(readAddress);
201 assertTrue(expectedAddress.equals(readAddress));
202 }
203
204 public static class WaitToBeMasterThread extends Thread {
205
206 ActiveMasterManager manager;
207 DummyMaster dummyMaster;
208 boolean isActiveMaster;
209
210 public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
211 this.dummyMaster = new DummyMaster(zk,address);
212 this.manager = this.dummyMaster.getActiveMasterManager();
213 isActiveMaster = false;
214 }
215
216 @Override
217 public void run() {
218 manager.blockUntilBecomingActiveMaster(100,
219 Mockito.mock(MonitoredTask.class));
220 LOG.info("Second master has become the active master!");
221 isActiveMaster = true;
222 }
223 }
224
225 public static class NodeDeletionListener extends ZooKeeperListener {
226 private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
227
228 private Semaphore lock;
229 private String node;
230
231 public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
232 super(watcher);
233 lock = new Semaphore(0);
234 this.node = node;
235 }
236
237 @Override
238 public void nodeDeleted(String path) {
239 if(path.equals(node)) {
240 LOG.debug("nodeDeleted(" + path + ")");
241 lock.release();
242 }
243 }
244
245 public void waitForDeletion() throws InterruptedException {
246 lock.acquire();
247 }
248 }
249
250
251
252
253 public static class DummyMaster implements Server {
254 private volatile boolean stopped;
255 private ClusterStatusTracker clusterStatusTracker;
256 private ActiveMasterManager activeMasterManager;
257
258 public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
259 this.clusterStatusTracker =
260 new ClusterStatusTracker(zk, this);
261 clusterStatusTracker.start();
262
263 this.activeMasterManager =
264 new ActiveMasterManager(zk, master, this);
265 zk.registerListener(activeMasterManager);
266 }
267
268 @Override
269 public void abort(final String msg, final Throwable t) {}
270
271 @Override
272 public boolean isAborted() {
273 return false;
274 }
275
276 @Override
277 public Configuration getConfiguration() {
278 return null;
279 }
280
281 @Override
282 public ZooKeeperWatcher getZooKeeper() {
283 return null;
284 }
285
286 @Override
287 public CoordinatedStateManager getCoordinatedStateManager() {
288 return null;
289 }
290
291 @Override
292 public ServerName getServerName() {
293 return null;
294 }
295
296 @Override
297 public boolean isStopped() {
298 return this.stopped;
299 }
300
301 @Override
302 public void stop(String why) {
303 this.stopped = true;
304 }
305
306 @Override
307 public ClusterConnection getConnection() {
308 return null;
309 }
310
311 @Override
312 public MetaTableLocator getMetaTableLocator() {
313 return null;
314 }
315
316 public ClusterStatusTracker getClusterStatusTracker() {
317 return clusterStatusTracker;
318 }
319
320 public ActiveMasterManager getActiveMasterManager() {
321 return activeMasterManager;
322 }
323
324 @Override
325 public ChoreService getChoreService() {
326 return null;
327 }
328 }
329 }