1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import java.io.IOException;
22 import java.util.concurrent.atomic.AtomicInteger;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HRegionLocation;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.MetaTableAccessor;
34 import org.apache.hadoop.hbase.RegionLocations;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.TableStateManager;
38 import org.apache.hadoop.hbase.client.BufferedMutator;
39 import org.apache.hadoop.hbase.client.Connection;
40 import org.apache.hadoop.hbase.client.Durability;
41 import org.apache.hadoop.hbase.client.NonceGenerator;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.MetaScanner;
44 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
45 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.master.HMaster;
48 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
49 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
50 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.MD5Hash;
55
56 import static org.junit.Assert.assertEquals;
57 import static org.junit.Assert.assertFalse;
58 import static org.junit.Assert.assertTrue;
59 import static org.junit.Assert.fail;
60
61 public class MasterProcedureTestingUtility {
62 private static final Log LOG = LogFactory.getLog(MasterProcedureTestingUtility.class);
63
64 private MasterProcedureTestingUtility() {
65 }
66
67 public static HTableDescriptor createHTD(final TableName tableName, final String... family) {
68 HTableDescriptor htd = new HTableDescriptor(tableName);
69 for (int i = 0; i < family.length; ++i) {
70 htd.addFamily(new HColumnDescriptor(family[i]));
71 }
72 return htd;
73 }
74
75 public static HRegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec,
76 final TableName tableName, final byte[][] splitKeys, String... family) throws IOException {
77 HTableDescriptor htd = createHTD(tableName, family);
78 HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, splitKeys);
79 long procId = ProcedureTestingUtility.submitAndWait(procExec,
80 new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
81 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
82 return regions;
83 }
84
85 public static void validateTableCreation(final HMaster master, final TableName tableName,
86 final HRegionInfo[] regions, String... family) throws IOException {
87 validateTableCreation(master, tableName, regions, true, family);
88 }
89
90 public static void validateTableCreation(final HMaster master, final TableName tableName,
91 final HRegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
92
93 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
94 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
95 assertTrue(fs.exists(tableDir));
96 FSUtils.logFileSystemState(fs, tableDir, LOG);
97 List<Path> allRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
98 for (int i = 0; i < regions.length; ++i) {
99 Path regionDir = new Path(tableDir, regions[i].getEncodedName());
100 assertTrue(regions[i] + " region dir does not exist", fs.exists(regionDir));
101 assertTrue(allRegionDirs.remove(regionDir));
102 List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir);
103 for (int j = 0; j < family.length; ++j) {
104 final Path familyDir = new Path(regionDir, family[j]);
105 if (hasFamilyDirs) {
106 assertTrue(family[j] + " family dir does not exist", fs.exists(familyDir));
107 assertTrue(allFamilyDirs.remove(familyDir));
108 } else {
109
110 if (!fs.exists(familyDir)) {
111 LOG.warn(family[j] + " family dir does not exist");
112 }
113 allFamilyDirs.remove(familyDir);
114 }
115 }
116 assertTrue("found extraneous families: " + allFamilyDirs, allFamilyDirs.isEmpty());
117 }
118 assertTrue("found extraneous regions: " + allRegionDirs, allRegionDirs.isEmpty());
119
120
121 assertTrue(MetaTableAccessor.tableExists(master.getConnection(), tableName));
122 assertEquals(regions.length, countMetaRegions(master, tableName));
123
124
125 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
126 assertTrue("table descriptor not found", htd != null);
127 for (int i = 0; i < family.length; ++i) {
128 assertTrue("family not found " + family[i], htd.getFamily(Bytes.toBytes(family[i])) != null);
129 }
130 assertEquals(family.length, htd.getFamilies().size());
131 }
132
133 public static void validateTableDeletion(final HMaster master, final TableName tableName,
134 final HRegionInfo[] regions, String... family) throws IOException {
135
136 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
137 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
138 assertFalse(fs.exists(tableDir));
139
140
141 assertFalse(MetaTableAccessor.tableExists(master.getConnection(), tableName));
142 assertEquals(0, countMetaRegions(master, tableName));
143
144
145 assertTrue("found htd of deleted table",
146 master.getTableDescriptors().get(tableName) == null);
147 }
148
149 private static int countMetaRegions(final HMaster master, final TableName tableName)
150 throws IOException {
151 final AtomicInteger actualRegCount = new AtomicInteger(0);
152 final MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
153 @Override
154 public boolean processRow(Result rowResult) throws IOException {
155 RegionLocations list = MetaTableAccessor.getRegionLocations(rowResult);
156 if (list == null) {
157 LOG.warn("No serialized HRegionInfo in " + rowResult);
158 return true;
159 }
160 HRegionLocation l = list.getRegionLocation();
161 if (l == null) {
162 return true;
163 }
164 if (!l.getRegionInfo().getTable().equals(tableName)) {
165 return false;
166 }
167 if (l.getRegionInfo().isOffline() || l.getRegionInfo().isSplit()) return true;
168 HRegionLocation[] locations = list.getRegionLocations();
169 for (HRegionLocation location : locations) {
170 if (location == null) continue;
171 ServerName serverName = location.getServerName();
172
173 if (serverName != null && serverName.getHostAndPort() != null) {
174 actualRegCount.incrementAndGet();
175 }
176 }
177 return true;
178 }
179 };
180 MetaScanner.metaScan(master.getConnection(), visitor, tableName);
181 return actualRegCount.get();
182 }
183
184 public static void validateTableIsEnabled(final HMaster master, final TableName tableName)
185 throws IOException {
186 TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
187 assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.ENABLED));
188 }
189
190 public static void validateTableIsDisabled(final HMaster master, final TableName tableName)
191 throws IOException {
192 TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
193 assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED));
194 }
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211 public static <TState> void testRecoveryAndDoubleExecution(
212 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
213 final int numSteps, final TState[] states) throws Exception {
214 ProcedureTestingUtility.waitProcedure(procExec, procId);
215 assertEquals(false, procExec.isRunning());
216 for (int i = 0; i < numSteps; ++i) {
217 LOG.info("Restart "+ i +" exec state: " + states[i]);
218 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
219 ProcedureTestingUtility.restart(procExec);
220 ProcedureTestingUtility.waitProcedure(procExec, procId);
221 }
222 assertEquals(true, procExec.isRunning());
223 ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
224 }
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242 public static <TState> void testRecoveryAndDoubleExecution(
243 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId)
244 throws Exception {
245 ProcedureTestingUtility.waitProcedure(procExec, procId);
246 assertEquals(false, procExec.isRunning());
247 while (!procExec.isFinished(procId)) {
248 ProcedureTestingUtility.restart(procExec);
249 ProcedureTestingUtility.waitProcedure(procExec, procId);
250 }
251 assertEquals(true, procExec.isRunning());
252 ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
253 }
254
255 public static <TState> void testRollbackAndDoubleExecution(
256 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
257 final int lastStep, final TState[] states) throws Exception {
258 ProcedureTestingUtility.waitProcedure(procExec, procId);
259
260
261
262
263
264 for (int i = 0; i < lastStep; ++i) {
265 LOG.info("Restart "+ i +" exec state: " + states[i]);
266 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
267 ProcedureTestingUtility.restart(procExec);
268 ProcedureTestingUtility.waitProcedure(procExec, procId);
269 }
270
271
272
273
274
275 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
276 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
277 procExec.registerListener(abortListener);
278 try {
279 for (int i = lastStep + 1; i >= 0; --i) {
280 LOG.info("Restart " + i +" rollback state: "+ states[i]);
281 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
282 ProcedureTestingUtility.restart(procExec);
283 ProcedureTestingUtility.waitProcedure(procExec, procId);
284 }
285 } finally {
286 assertTrue(procExec.unregisterListener(abortListener));
287 }
288
289 ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
290 }
291
292 public static <TState> void testRollbackAndDoubleExecutionAfterPONR(
293 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
294 final int lastStep, final TState[] states) throws Exception {
295 ProcedureTestingUtility.waitProcedure(procExec, procId);
296
297
298
299
300
301 for (int i = 0; i < lastStep; ++i) {
302 LOG.info("Restart "+ i +" exec state: " + states[i]);
303 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
304 ProcedureTestingUtility.restart(procExec);
305 ProcedureTestingUtility.waitProcedure(procExec, procId);
306 }
307
308
309 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
310 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
311 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
312 procExec.registerListener(abortListener);
313 try {
314 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
315 ProcedureTestingUtility.restart(procExec);
316 LOG.info("Restart and execute");
317 ProcedureTestingUtility.waitProcedure(procExec, procId);
318 } finally {
319 assertTrue(procExec.unregisterListener(abortListener));
320 }
321
322 assertEquals(true, procExec.isRunning());
323 ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
324 }
325
326 public static <TState> void testRollbackRetriableFailure(
327 final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId,
328 final int lastStep, final TState[] states) throws Exception {
329 ProcedureTestingUtility.waitProcedure(procExec, procId);
330
331
332
333
334
335 for (int i = 0; i < lastStep; ++i) {
336 LOG.info("Restart "+ i +" exec state: " + states[i]);
337 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
338 ProcedureTestingUtility.restart(procExec);
339 ProcedureTestingUtility.waitProcedure(procExec, procId);
340 }
341
342
343 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
344 MasterProcedureTestingUtility.InjectAbortOnLoadListener abortListener =
345 new MasterProcedureTestingUtility.InjectAbortOnLoadListener(procExec);
346 procExec.registerListener(abortListener);
347 try {
348 ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
349 ProcedureTestingUtility.restart(procExec);
350 LOG.info("Restart and rollback");
351 ProcedureTestingUtility.waitProcedure(procExec, procId);
352 } finally {
353 assertTrue(procExec.unregisterListener(abortListener));
354 }
355
356 ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
357 }
358
359 public static void validateColumnFamilyAddition(final HMaster master, final TableName tableName,
360 final String family) throws IOException {
361 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
362 assertTrue(htd != null);
363 assertTrue(htd.hasFamily(family.getBytes()));
364 }
365
366 public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
367 final String family) throws IOException {
368
369 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
370 assertTrue(htd != null);
371 assertFalse(htd.hasFamily(family.getBytes()));
372
373
374 final FileSystem fs = master.getMasterFileSystem().getFileSystem();
375 final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
376 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
377 final Path familyDir = new Path(regionDir, family);
378 assertFalse(family + " family dir should not exist", fs.exists(familyDir));
379 }
380 }
381
382 public static void validateColumnFamilyModification(final HMaster master,
383 final TableName tableName, final String family, HColumnDescriptor columnDescriptor)
384 throws IOException {
385 HTableDescriptor htd = master.getTableDescriptors().get(tableName);
386 assertTrue(htd != null);
387
388 HColumnDescriptor hcfd = htd.getFamily(family.getBytes());
389 assertTrue(hcfd.equals(columnDescriptor));
390 }
391
392 public static void loadData(final Connection connection, final TableName tableName,
393 int rows, final byte[][] splitKeys, final String... sfamilies) throws IOException {
394 byte[][] families = new byte[sfamilies.length][];
395 for (int i = 0; i < families.length; ++i) {
396 families[i] = Bytes.toBytes(sfamilies[i]);
397 }
398
399 BufferedMutator mutator = connection.getBufferedMutator(tableName);
400
401
402 assertTrue(rows >= splitKeys.length);
403 for (byte[] k: splitKeys) {
404 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
405 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
406 mutator.mutate(createPut(families, key, value));
407 rows--;
408 }
409
410
411 while (rows-- > 0) {
412 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
413 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
414 mutator.mutate(createPut(families, key, value));
415 }
416 mutator.flush();
417 }
418
419 private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
420 byte[] q = Bytes.toBytes("q");
421 Put put = new Put(key);
422 put.setDurability(Durability.SKIP_WAL);
423 for (byte[] family: families) {
424 put.add(family, q, value);
425 }
426 return put;
427 }
428
429 public static long generateNonceGroup(final HMaster master) {
430 return master.getConnection().getNonceGenerator().getNonceGroup();
431 }
432
433 public static long generateNonce(final HMaster master) {
434 return master.getConnection().getNonceGenerator().newNonce();
435 }
436
437 public static class InjectAbortOnLoadListener
438 implements ProcedureExecutor.ProcedureExecutorListener {
439 private final ProcedureExecutor<MasterProcedureEnv> procExec;
440
441 public InjectAbortOnLoadListener(final ProcedureExecutor<MasterProcedureEnv> procExec) {
442 this.procExec = procExec;
443 }
444
445 @Override
446 public void procedureLoaded(long procId) {
447 procExec.abort(procId);
448 }
449
450 @Override
451 public void procedureAdded(long procId) {
452
453 @Override
454 public void procedureFinished(long procId) {
455 }
456 }