1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Matchers.anyInt;
25 import static org.mockito.Matchers.eq;
26 import static org.mockito.Mockito.doNothing;
27 import static org.mockito.Mockito.doThrow;
28 import static org.mockito.Mockito.spy;
29 import static org.mockito.Mockito.when;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.List;
34
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.Cell;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.Server;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
48 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
49 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
50 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
51 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
52 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
53 import org.apache.hadoop.hbase.testclassification.SmallTests;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.FSUtils;
56 import org.apache.hadoop.hbase.util.PairOfSameType;
57 import org.apache.hadoop.hbase.wal.WALFactory;
58 import org.apache.zookeeper.KeeperException;
59 import org.junit.After;
60 import org.junit.Before;
61 import org.junit.Test;
62 import org.junit.experimental.categories.Category;
63 import org.mockito.Mockito;
64
65 import com.google.common.collect.ImmutableList;
66
67
68
69
70
71 @Category(SmallTests.class)
72 public class TestSplitTransaction {
73 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
74 private final Path testdir =
75 TEST_UTIL.getDataTestDir(this.getClass().getName());
76 private HRegion parent;
77 private WALFactory wals;
78 private FileSystem fs;
79 private static final byte [] STARTROW = new byte [] {'a', 'a', 'a'};
80
81 private static final byte [] ENDROW = new byte [] {'{', '{', '{'};
82 private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'};
83 private static final byte [] CF = HConstants.CATALOG_FAMILY;
84
85 private static boolean preRollBackCalled = false;
86 private static boolean postRollBackCalled = false;
87
88 @Before public void setup() throws IOException {
89 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
90 TEST_UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
91 this.fs.delete(this.testdir, true);
92 final Configuration walConf = new Configuration(TEST_UTIL.getConfiguration());
93 FSUtils.setRootDir(walConf, this.testdir);
94 this.wals = new WALFactory(walConf, null, this.getClass().getName());
95
96 this.parent = createRegion(this.testdir, this.wals);
97 RegionCoprocessorHost host = new RegionCoprocessorHost(this.parent, null, TEST_UTIL.getConfiguration());
98 this.parent.setCoprocessorHost(host);
99 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
100 }
101
102 @After public void teardown() throws IOException {
103 if (this.parent != null && !this.parent.isClosed()) this.parent.close();
104 Path regionDir = this.parent.getRegionFileSystem().getRegionDir();
105 if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
106 throw new IOException("Failed delete of " + regionDir);
107 }
108 if (this.wals != null) {
109 this.wals.close();
110 }
111 this.fs.delete(this.testdir, true);
112 }
113
114 @Test public void testFailAfterPONR() throws IOException, KeeperException {
115 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
116 assertTrue(rowcount > 0);
117 int parentRowCount = countRows(this.parent);
118 assertEquals(rowcount, parentRowCount);
119
120
121 SplitTransactionImpl st = prepareGOOD_SPLIT_ROW();
122 SplitTransactionImpl spiedUponSt = spy(st);
123 Mockito
124 .doThrow(new MockedFailedDaughterOpen())
125 .when(spiedUponSt)
126 .openDaughterRegion((Server) Mockito.anyObject(),
127 (HRegion) Mockito.anyObject());
128
129
130 boolean expectedException = false;
131 Server mockServer = Mockito.mock(Server.class);
132 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
133 try {
134 spiedUponSt.execute(mockServer, null);
135 } catch (IOException e) {
136 if (e.getCause() != null &&
137 e.getCause() instanceof MockedFailedDaughterOpen) {
138 expectedException = true;
139 }
140 }
141 assertTrue(expectedException);
142
143 assertFalse(spiedUponSt.rollback(null, null));
144
145
146
147 Path tableDir = this.parent.getRegionFileSystem().getTableDir();
148 Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName());
149 Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName());
150 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir));
151 assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir));
152 }
153
154
155
156
157
158 @Test public void testPrepare() throws IOException {
159 prepareGOOD_SPLIT_ROW();
160 }
161
162 private SplitTransactionImpl prepareGOOD_SPLIT_ROW() throws IOException {
163 return prepareGOOD_SPLIT_ROW(this.parent);
164 }
165
166 private SplitTransactionImpl prepareGOOD_SPLIT_ROW(final HRegion parentRegion)
167 throws IOException {
168 SplitTransactionImpl st = new SplitTransactionImpl(parentRegion, GOOD_SPLIT_ROW);
169 assertTrue(st.prepare());
170 return st;
171 }
172
173
174
175
176 @Test public void testPrepareWithRegionsWithReference() throws IOException {
177 HStore storeMock = Mockito.mock(HStore.class);
178 when(storeMock.hasReferences()).thenReturn(true);
179 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
180 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
181 this.parent.stores.put(Bytes.toBytes(""), storeMock);
182
183 SplitTransactionImpl st = new SplitTransactionImpl(this.parent, GOOD_SPLIT_ROW);
184
185 assertFalse("a region should not be splittable if it has instances of store file references",
186 st.prepare());
187 }
188
189
190
191
192 @Test public void testPrepareWithBadSplitRow() throws IOException {
193
194 SplitTransactionImpl st = new SplitTransactionImpl(this.parent, STARTROW);
195 assertFalse(st.prepare());
196 st = new SplitTransactionImpl(this.parent, HConstants.EMPTY_BYTE_ARRAY);
197 assertFalse(st.prepare());
198 st = new SplitTransactionImpl(this.parent, new byte [] {'A', 'A', 'A'});
199 assertFalse(st.prepare());
200 st = new SplitTransactionImpl(this.parent, ENDROW);
201 assertFalse(st.prepare());
202 }
203
204 @Test public void testPrepareWithClosedRegion() throws IOException {
205 this.parent.close();
206 SplitTransactionImpl st = new SplitTransactionImpl(this.parent, GOOD_SPLIT_ROW);
207 assertFalse(st.prepare());
208 }
209
210 @Test public void testWholesomeSplit() throws IOException {
211 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF, true);
212 assertTrue(rowcount > 0);
213 int parentRowCount = countRows(this.parent);
214 assertEquals(rowcount, parentRowCount);
215
216
217
218 CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
219 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
220
221
222 SplitTransactionImpl st = prepareGOOD_SPLIT_ROW();
223
224
225 Server mockServer = Mockito.mock(Server.class);
226 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
227 PairOfSameType<Region> daughters = st.execute(mockServer, null);
228
229 assertTrue(this.fs.exists(this.parent.getRegionFileSystem().getSplitsDir()));
230
231 assertTrue(this.parent.isClosed());
232
233
234
235 assertEquals(0, this.fs.listStatus(this.parent.getRegionFileSystem().getSplitsDir()).length);
236
237 assertTrue(Bytes.equals(parent.getRegionInfo().getStartKey(),
238 daughters.getFirst().getRegionInfo().getStartKey()));
239 assertTrue(Bytes.equals(GOOD_SPLIT_ROW, daughters.getFirst().getRegionInfo().getEndKey()));
240 assertTrue(Bytes.equals(daughters.getSecond().getRegionInfo().getStartKey(), GOOD_SPLIT_ROW));
241 assertTrue(Bytes.equals(parent.getRegionInfo().getEndKey(),
242 daughters.getSecond().getRegionInfo().getEndKey()));
243
244 int daughtersRowCount = 0;
245 for (Region openRegion: daughters) {
246 try {
247 int count = countRows(openRegion);
248 assertTrue(count > 0 && count != rowcount);
249 daughtersRowCount += count;
250 } finally {
251 HRegion.closeHRegion((HRegion)openRegion);
252 }
253 }
254 assertEquals(rowcount, daughtersRowCount);
255
256 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
257 }
258
259 @Test
260 public void testCountReferencesFailsSplit() throws IOException {
261 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
262 assertTrue(rowcount > 0);
263 int parentRowCount = countRows(this.parent);
264 assertEquals(rowcount, parentRowCount);
265
266
267 HRegion spiedRegion = spy(this.parent);
268 SplitTransactionImpl st = prepareGOOD_SPLIT_ROW(spiedRegion);
269 SplitTransactionImpl spiedUponSt = spy(st);
270 doThrow(new IOException("Failing split. Expected reference file count isn't equal."))
271 .when(spiedUponSt).assertReferenceFileCount(anyInt(),
272 eq(new Path(this.parent.getRegionFileSystem().getTableDir(),
273 st.getSecondDaughter().getEncodedName())));
274
275
276 boolean expectedException = false;
277 Server mockServer = Mockito.mock(Server.class);
278 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
279 try {
280 spiedUponSt.execute(mockServer, null);
281 } catch (IOException e) {
282 expectedException = true;
283 }
284 assertTrue(expectedException);
285 }
286
287
288 @Test public void testRollback() throws IOException {
289 final int rowcount = TEST_UTIL.loadRegion(this.parent, CF);
290 assertTrue(rowcount > 0);
291 int parentRowCount = countRows(this.parent);
292 assertEquals(rowcount, parentRowCount);
293
294
295 HRegion spiedRegion = spy(this.parent);
296 SplitTransactionImpl st = prepareGOOD_SPLIT_ROW(spiedRegion);
297 SplitTransactionImpl spiedUponSt = spy(st);
298 doNothing().when(spiedUponSt).assertReferenceFileCount(anyInt(),
299 eq(parent.getRegionFileSystem().getSplitsDir(st.getFirstDaughter())));
300 when(spiedRegion.createDaughterRegionFromSplits(spiedUponSt.getSecondDaughter())).
301 thenThrow(new MockedFailedDaughterCreation());
302
303 boolean expectedException = false;
304 Server mockServer = Mockito.mock(Server.class);
305 when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration());
306 try {
307 spiedUponSt.execute(mockServer, null);
308 } catch (MockedFailedDaughterCreation e) {
309 expectedException = true;
310 }
311 assertTrue(expectedException);
312
313 assertTrue(spiedUponSt.rollback(null, null));
314
315
316 int parentRowCount2 = countRows(this.parent);
317 assertEquals(parentRowCount, parentRowCount2);
318
319
320 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getFirstDaughter())));
321 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir, st.getSecondDaughter())));
322 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
323
324
325 assertTrue(st.prepare());
326 PairOfSameType<Region> daughters = st.execute(mockServer, null);
327
328 int daughtersRowCount = 0;
329 for (Region openRegion: daughters) {
330 try {
331 int count = countRows(openRegion);
332 assertTrue(count > 0 && count != rowcount);
333 daughtersRowCount += count;
334 } finally {
335 HRegion.closeHRegion((HRegion)openRegion);
336 }
337 }
338 assertEquals(rowcount, daughtersRowCount);
339
340 assertTrue(!this.parent.lock.writeLock().isHeldByCurrentThread());
341 assertTrue("Rollback hooks should be called.", wasRollBackHookCalled());
342 }
343
344 private boolean wasRollBackHookCalled(){
345 return (preRollBackCalled && postRollBackCalled);
346 }
347
348
349
350
351 @SuppressWarnings("serial")
352 private class MockedFailedDaughterCreation extends IOException {}
353 private class MockedFailedDaughterOpen extends IOException {}
354
355 private int countRows(final Region r) throws IOException {
356 int rowcount = 0;
357 InternalScanner scanner = r.getScanner(new Scan());
358 try {
359 List<Cell> kvs = new ArrayList<Cell>();
360 boolean hasNext = true;
361 while (hasNext) {
362 hasNext = scanner.next(kvs);
363 if (!kvs.isEmpty()) rowcount++;
364 }
365 } finally {
366 scanner.close();
367 }
368 return rowcount;
369 }
370
371 HRegion createRegion(final Path testdir, final WALFactory wals)
372 throws IOException {
373
374
375 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
376 HColumnDescriptor hcd = new HColumnDescriptor(CF);
377 htd.addFamily(hcd);
378 HRegionInfo hri = new HRegionInfo(htd.getTableName(), STARTROW, ENDROW);
379 HRegion r = HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
380 HRegion.closeHRegion(r);
381 return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()),
382 TEST_UTIL.getConfiguration());
383 }
384
385 public static class CustomObserver extends BaseRegionObserver{
386 @Override
387 public void preRollBackSplit(
388 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
389 preRollBackCalled = true;
390 }
391
392 @Override
393 public void postRollBackSplit(
394 ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
395 postRollBackCalled = true;
396 }
397 }
398
399 }
400