View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.handler;
20  
21  import static org.junit.Assert.*;
22  
23  import java.io.IOException;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.*;
28  import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
29  import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
30  import org.apache.hadoop.hbase.executor.EventType;
31  import org.apache.hadoop.hbase.regionserver.HRegion;
32  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
33  import org.apache.hadoop.hbase.testclassification.MediumTests;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.MockServer;
36  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
38  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
39  import org.apache.zookeeper.KeeperException;
40  import org.apache.zookeeper.KeeperException.NodeExistsException;
41  import org.junit.AfterClass;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * Test of the {@link OpenRegionHandler}.
49   */
50  @Category(MediumTests.class)
51  public class TestOpenRegionHandler {
52    static final Log LOG = LogFactory.getLog(TestOpenRegionHandler.class);
53    private final static HBaseTestingUtility HTU = HBaseTestingUtility.createLocalHTU();
54    private static HTableDescriptor TEST_HTD;
55    private HRegionInfo TEST_HRI;
56  
57    private int testIndex = 0;
58  
59    @BeforeClass public static void before() throws Exception {
60      HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
61      HTU.startMiniZKCluster();
62      TEST_HTD = new HTableDescriptor(TableName.valueOf("TestOpenRegionHandler.java"));
63    }
64  
65    @AfterClass public static void after() throws IOException {
66      TEST_HTD = null;
67      HTU.shutdownMiniZKCluster();
68    }
69  
70    /**
71     * Before each test, use a different HRI, so the different tests
72     * don't interfere with each other. This allows us to use just
73     * a single ZK cluster for the whole suite.
74     */
75    @Before
76    public void setupHRI() {
77      TEST_HRI = new HRegionInfo(TEST_HTD.getTableName(),
78        Bytes.toBytes(testIndex),
79        Bytes.toBytes(testIndex + 1));
80      testIndex++;
81    }
82  
83    /**
84     * Test the openregionhandler can deal with its znode being yanked out from
85     * under it.
86     * @see <a href="https://issues.apache.org/jira/browse/HBASE-3627">HBASE-3627</a>
87     * @throws IOException
88     * @throws NodeExistsException
89     * @throws KeeperException
90     */
91    @Test public void testYankingRegionFromUnderIt()
92    throws IOException, NodeExistsException, KeeperException {
93      final Server server = new MockServer(HTU);
94      final RegionServerServices rss = HTU.createMockRegionServerService();
95  
96      HTableDescriptor htd = TEST_HTD;
97      final HRegionInfo hri = TEST_HRI;
98      HRegion region =
99           HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU
100             .getConfiguration(), htd);
101     assertNotNull(region);
102     try {
103       ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
104       csm.initialize(server);
105       csm.start();
106 
107       ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
108         new ZkOpenRegionCoordination.ZkOpenRegionDetails();
109       zkCrd.setServerName(server.getServerName());
110 
111       OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri,
112         htd, -1, csm.getOpenRegionCoordination(), zkCrd) {
113         HRegion openRegion() {
114           // Open region first, then remove znode as though it'd been hijacked.
115           HRegion region = super.openRegion();
116 
117           // Don't actually open region BUT remove the znode as though it'd
118           // been hijacked on us.
119           ZooKeeperWatcher zkw = this.server.getZooKeeper();
120           String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
121           try {
122             ZKUtil.deleteNodeFailSilent(zkw, node);
123           } catch (KeeperException e) {
124             throw new RuntimeException("Ugh failed delete of " + node, e);
125           }
126           return region;
127         }
128       };
129       rss.getRegionsInTransitionInRS().put(
130         hri.getEncodedNameAsBytes(), Boolean.TRUE);
131       // Call process without first creating OFFLINE region in zk, see if
132       // exception or just quiet return (expected).
133       handler.process();
134       rss.getRegionsInTransitionInRS().put(
135         hri.getEncodedNameAsBytes(), Boolean.TRUE);
136       ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
137       // Call process again but this time yank the zk znode out from under it
138       // post OPENING; again will expect it to come back w/o NPE or exception.
139       handler.process();
140     } finally {
141       HRegion.closeHRegion(region);
142     }
143   }
144   
145   /**
146    * Test the openregionhandler can deal with perceived failure of transitioning to OPENED state
147    * due to intermittent zookeeper malfunctioning.
148    * @see <a href="https://issues.apache.org/jira/browse/HBASE-9387">HBASE-9387</a>
149    * @throws IOException
150    * @throws NodeExistsException
151    * @throws KeeperException
152    */
153   @Test
154   public void testRegionServerAbortionDueToFailureTransitioningToOpened()
155       throws IOException, NodeExistsException, KeeperException {
156     final Server server = new MockServer(HTU);
157     final RegionServerServices rss = HTU.createMockRegionServerService();
158 
159     HTableDescriptor htd = TEST_HTD;
160     final HRegionInfo hri = TEST_HRI;
161     HRegion region =
162          HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU
163             .getConfiguration(), htd);
164     assertNotNull(region);
165     try {
166 
167       ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
168       csm.initialize(server);
169       csm.start();
170 
171       ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
172         new ZkOpenRegionCoordination.ZkOpenRegionDetails();
173       zkCrd.setServerName(server.getServerName());
174 
175       ZkOpenRegionCoordination openRegionCoordination =
176         new ZkOpenRegionCoordination(csm, server.getZooKeeper()) {
177         @Override
178         public boolean transitionToOpened(final HRegion r, OpenRegionDetails ord)
179             throws IOException {
180           // remove znode simulating intermittent zookeeper connection issue
181           ZooKeeperWatcher zkw = server.getZooKeeper();
182           String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
183           try {
184             ZKUtil.deleteNodeFailSilent(zkw, node);
185           } catch (KeeperException e) {
186             throw new RuntimeException("Ugh failed delete of " + node, e);
187           }
188           // then try to transition to OPENED
189           return super.transitionToOpened(r, ord);
190         }
191       };
192 
193       OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd,
194         -1, openRegionCoordination, zkCrd);
195       rss.getRegionsInTransitionInRS().put(
196         hri.getEncodedNameAsBytes(), Boolean.TRUE);
197       // Call process without first creating OFFLINE region in zk, see if
198       // exception or just quiet return (expected).
199       handler.process();
200       rss.getRegionsInTransitionInRS().put(
201         hri.getEncodedNameAsBytes(), Boolean.TRUE);
202       ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
203       // Call process again but this time yank the zk znode out from under it
204       // post OPENING; again will expect it to come back w/o NPE or exception.
205       handler.process();
206     } catch (IOException ioe) {
207     } finally {
208       HRegion.closeHRegion(region);
209     }
210     // Region server is expected to abort due to OpenRegionHandler perceiving transitioning
211     // to OPENED as failed
212     // This was corresponding to the second handler.process() call above.
213     assertTrue("region server should have aborted", server.isAborted());
214   }
215   
216   @Test
217   public void testFailedOpenRegion() throws Exception {
218     Server server = new MockServer(HTU);
219     RegionServerServices rsServices = HTU.createMockRegionServerService();
220 
221     // Create it OFFLINE, which is what it expects
222     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
223 
224     ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
225     csm.initialize(server);
226     csm.start();
227 
228     ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
229       new ZkOpenRegionCoordination.ZkOpenRegionDetails();
230     zkCrd.setServerName(server.getServerName());
231 
232     // Create the handler
233     OpenRegionHandler handler =
234       new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD, -1,
235         csm.getOpenRegionCoordination(), zkCrd) {
236         @Override
237         HRegion openRegion() {
238           // Fake failure of opening a region due to an IOE, which is caught
239           return null;
240         }
241     };
242     rsServices.getRegionsInTransitionInRS().put(
243       TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
244     handler.process();
245 
246     // Handler should have transitioned it to FAILED_OPEN
247     RegionTransition rt = RegionTransition.parseFrom(
248       ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName()));
249     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
250   }
251   
252   @Test
253   public void testFailedUpdateMeta() throws Exception {
254     Server server = new MockServer(HTU);
255     RegionServerServices rsServices = HTU.createMockRegionServerService();
256 
257     // Create it OFFLINE, which is what it expects
258     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
259 
260     // Create the handler
261     ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
262     csm.initialize(server);
263     csm.start();
264 
265     ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
266       new ZkOpenRegionCoordination.ZkOpenRegionDetails();
267     zkCrd.setServerName(server.getServerName());
268 
269     OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD,
270       -1, csm.getOpenRegionCoordination(), zkCrd) {
271         @Override
272         boolean updateMeta(final HRegion r, long masterSystemTime) {
273           // Fake failure of updating META
274           return false;
275         }
276     };
277     rsServices.getRegionsInTransitionInRS().put(
278       TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
279     handler.process();
280 
281     // Handler should have transitioned it to FAILED_OPEN
282     RegionTransition rt = RegionTransition.parseFrom(
283       ZKAssign.getData(server.getZooKeeper(), TEST_HRI.getEncodedName()));
284     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
285   }
286   
287   @Test
288   public void testTransitionToFailedOpenEvenIfCleanupFails() throws Exception {
289     Server server = new MockServer(HTU);
290     RegionServerServices rsServices = HTU.createMockRegionServerService();
291     // Create it OFFLINE, which is what it expects
292     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
293     // Create the handler
294     ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
295     csm.initialize(server);
296     csm.start();
297 
298     ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
299       new ZkOpenRegionCoordination.ZkOpenRegionDetails();
300     zkCrd.setServerName(server.getServerName());
301 
302     OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD,
303       -1, csm.getOpenRegionCoordination(), zkCrd) {
304       @Override
305       boolean updateMeta(HRegion r, long masterSystemTime) {
306         return false;
307       };
308 
309       @Override
310       void cleanupFailedOpen(HRegion region) throws IOException {
311         throw new IOException("FileSystem got closed.");
312       }
313     };
314     rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
315     try {
316       handler.process();
317     } catch (Exception e) {
318       // Ignore the IOException that we have thrown from cleanupFailedOpen
319     }
320     RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(),
321         TEST_HRI.getEncodedName()));
322     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
323   }
324 
325   @Test
326   public void testTransitionToFailedOpenFromOffline() throws Exception {
327     Server server = new MockServer(HTU);
328     RegionServerServices rsServices = HTU.createMockRegionServerService(server.getServerName());
329     // Create it OFFLINE, which is what it expects
330     ZKAssign.createNodeOffline(server.getZooKeeper(), TEST_HRI, server.getServerName());
331     // Create the handler
332     ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager();
333     csm.initialize(server);
334     csm.start();
335 
336     ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
337       new ZkOpenRegionCoordination.ZkOpenRegionDetails();
338     zkCrd.setServerName(server.getServerName());
339 
340     ZkOpenRegionCoordination openRegionCoordination =
341       new ZkOpenRegionCoordination(csm, server.getZooKeeper()) {
342       @Override
343       public boolean transitionFromOfflineToOpening(HRegionInfo regionInfo,
344                                                     OpenRegionDetails ord) {
345         return false;
346       }
347     };
348 
349     OpenRegionHandler handler = new OpenRegionHandler(server, rsServices, TEST_HRI, TEST_HTD,
350       -1, openRegionCoordination, zkCrd);
351     rsServices.getRegionsInTransitionInRS().put(TEST_HRI.getEncodedNameAsBytes(), Boolean.TRUE);
352 
353     handler.process();
354 
355     RegionTransition rt = RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(),
356         TEST_HRI.getEncodedName()));
357     assertEquals(EventType.RS_ZK_REGION_FAILED_OPEN, rt.getEventType());
358   }
359 
360 }
361