View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.concurrent.Semaphore;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.ChoreService;
32  import org.apache.hadoop.hbase.CoordinatedStateManager;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.testclassification.MediumTests;
35  import org.apache.hadoop.hbase.Server;
36  import org.apache.hadoop.hbase.ServerName;
37  import org.apache.hadoop.hbase.client.ClusterConnection;
38  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
39  import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
40  import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
41  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
42  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
43  import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
44  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
45  import org.apache.zookeeper.KeeperException;
46  import org.junit.AfterClass;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  import org.mockito.Mockito;
51  
52  /**
53   * Test the {@link ActiveMasterManager}.
54   */
55  @Category(MediumTests.class)
56  public class TestActiveMasterManager {
57    private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
58    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59  
60    @BeforeClass
61    public static void setUpBeforeClass() throws Exception {
62      TEST_UTIL.startMiniZKCluster();
63    }
64  
65    @AfterClass
66    public static void tearDownAfterClass() throws Exception {
67      TEST_UTIL.shutdownMiniZKCluster();
68    }
69  
70    @Test public void testRestartMaster() throws IOException, KeeperException {
71      ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
72        "testActiveMasterManagerFromZK", null, true);
73      try {
74        ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
75        ZKUtil.deleteNode(zk, zk.clusterStateZNode);
76      } catch(KeeperException.NoNodeException nne) {}
77  
78      // Create the master node with a dummy address
79      ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
80      // Should not have a master yet
81      DummyMaster dummyMaster = new DummyMaster(zk,master);
82      ClusterStatusTracker clusterStatusTracker =
83        dummyMaster.getClusterStatusTracker();
84      ActiveMasterManager activeMasterManager =
85        dummyMaster.getActiveMasterManager();
86      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
87  
88      // First test becoming the active master uninterrupted
89      MonitoredTask status = Mockito.mock(MonitoredTask.class);
90      clusterStatusTracker.setClusterUp();
91  
92      activeMasterManager.blockUntilBecomingActiveMaster(100, status);
93      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
94      assertMaster(zk, master);
95  
96      // Now pretend master restart
97      DummyMaster secondDummyMaster = new DummyMaster(zk,master);
98      ActiveMasterManager secondActiveMasterManager =
99        secondDummyMaster.getActiveMasterManager();
100     assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
101     activeMasterManager.blockUntilBecomingActiveMaster(100, status);
102     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
103     assertMaster(zk, master);
104   }
105 
106   /**
107    * Unit tests that uses ZooKeeper but does not use the master-side methods
108    * but rather acts directly on ZK.
109    * @throws Exception
110    */
111   @Test
112   public void testActiveMasterManagerFromZK() throws Exception {
113     ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
114       "testActiveMasterManagerFromZK", null, true);
115     try {
116       ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
117       ZKUtil.deleteNode(zk, zk.clusterStateZNode);
118     } catch(KeeperException.NoNodeException nne) {}
119 
120     // Create the master node with a dummy address
121     ServerName firstMasterAddress =
122         ServerName.valueOf("localhost", 1, System.currentTimeMillis());
123     ServerName secondMasterAddress =
124         ServerName.valueOf("localhost", 2, System.currentTimeMillis());
125 
126     // Should not have a master yet
127     DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
128     ActiveMasterManager activeMasterManager =
129       ms1.getActiveMasterManager();
130     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
131 
132     // First test becoming the active master uninterrupted
133     ClusterStatusTracker clusterStatusTracker =
134       ms1.getClusterStatusTracker();
135     clusterStatusTracker.setClusterUp();
136     activeMasterManager.blockUntilBecomingActiveMaster(100, 
137         Mockito.mock(MonitoredTask.class));
138     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
139     assertMaster(zk, firstMasterAddress);
140 
141     // New manager will now try to become the active master in another thread
142     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
143     t.start();
144     // Wait for this guy to figure out there is another active master
145     // Wait for 1 second at most
146     int sleeps = 0;
147     while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
148       Thread.sleep(10);
149       sleeps++;
150     }
151 
152     // Both should see that there is an active master
153     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
154     assertTrue(t.manager.clusterHasActiveMaster.get());
155     // But secondary one should not be the active master
156     assertFalse(t.isActiveMaster);
157 
158     // Close the first server and delete it's master node
159     ms1.stop("stopping first server");
160 
161     // Use a listener to capture when the node is actually deleted
162     NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
163     zk.registerListener(listener);
164 
165     LOG.info("Deleting master node");
166     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
167 
168     // Wait for the node to be deleted
169     LOG.info("Waiting for active master manager to be notified");
170     listener.waitForDeletion();
171     LOG.info("Master node deleted");
172 
173     // Now we expect the secondary manager to have and be the active master
174     // Wait for 1 second at most
175     sleeps = 0;
176     while(!t.isActiveMaster && sleeps < 100) {
177       Thread.sleep(10);
178       sleeps++;
179     }
180     LOG.debug("Slept " + sleeps + " times");
181 
182     assertTrue(t.manager.clusterHasActiveMaster.get());
183     assertTrue(t.isActiveMaster);
184 
185     LOG.info("Deleting master node");
186     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
187   }
188 
189   /**
190    * Assert there is an active master and that it has the specified address.
191    * @param zk
192    * @param thisMasterAddress
193    * @throws KeeperException
194    * @throws IOException 
195    */
196   private void assertMaster(ZooKeeperWatcher zk,
197       ServerName expectedAddress)
198   throws KeeperException, IOException {
199     ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
200     assertNotNull(readAddress);
201     assertTrue(expectedAddress.equals(readAddress));
202   }
203 
204   public static class WaitToBeMasterThread extends Thread {
205 
206     ActiveMasterManager manager;
207     DummyMaster dummyMaster;
208     boolean isActiveMaster;
209 
210     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
211       this.dummyMaster = new DummyMaster(zk,address);
212       this.manager = this.dummyMaster.getActiveMasterManager();
213       isActiveMaster = false;
214     }
215 
216     @Override
217     public void run() {
218       manager.blockUntilBecomingActiveMaster(100,
219           Mockito.mock(MonitoredTask.class));
220       LOG.info("Second master has become the active master!");
221       isActiveMaster = true;
222     }
223   }
224 
225   public static class NodeDeletionListener extends ZooKeeperListener {
226     private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
227 
228     private Semaphore lock;
229     private String node;
230 
231     public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
232       super(watcher);
233       lock = new Semaphore(0);
234       this.node = node;
235     }
236 
237     @Override
238     public void nodeDeleted(String path) {
239       if(path.equals(node)) {
240         LOG.debug("nodeDeleted(" + path + ")");
241         lock.release();
242       }
243     }
244 
245     public void waitForDeletion() throws InterruptedException {
246       lock.acquire();
247     }
248   }
249 
250   /**
251    * Dummy Master Implementation.
252    */
253   public static class DummyMaster implements Server {
254     private volatile boolean stopped;
255     private ClusterStatusTracker clusterStatusTracker;
256     private ActiveMasterManager activeMasterManager;
257 
258     public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
259       this.clusterStatusTracker =
260         new ClusterStatusTracker(zk, this);
261       clusterStatusTracker.start();
262 
263       this.activeMasterManager =
264         new ActiveMasterManager(zk, master, this);
265       zk.registerListener(activeMasterManager);
266     }
267 
268     @Override
269     public void abort(final String msg, final Throwable t) {}
270     
271     @Override
272     public boolean isAborted() {
273       return false;
274     }
275 
276     @Override
277     public Configuration getConfiguration() {
278       return null;
279     }
280 
281     @Override
282     public ZooKeeperWatcher getZooKeeper() {
283       return null;
284     }
285 
286     @Override
287     public CoordinatedStateManager getCoordinatedStateManager() {
288       return null;
289     }
290 
291     @Override
292     public ServerName getServerName() {
293       return null;
294     }
295 
296     @Override
297     public boolean isStopped() {
298       return this.stopped;
299     }
300 
301     @Override
302     public void stop(String why) {
303       this.stopped = true;
304     }
305 
306     @Override
307     public ClusterConnection getConnection() {
308       return null;
309     }
310 
311     @Override
312     public MetaTableLocator getMetaTableLocator() {
313       return null;
314     }
315 
316     public ClusterStatusTracker getClusterStatusTracker() {
317       return clusterStatusTracker;
318     }
319 
320     public ActiveMasterManager getActiveMasterManager() {
321       return activeMasterManager;
322     }
323 
324     @Override
325     public ChoreService getChoreService() {
326       return null;
327     }
328   }
329 }