1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
35 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
36 import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
37 import org.apache.hadoop.hbase.testclassification.MediumTests;
38 import org.junit.After;
39 import org.junit.AfterClass;
40 import org.junit.Before;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45 @Category(MediumTests.class)
46 public class TestModifyTableProcedure {
47 private static final Log LOG = LogFactory.getLog(TestModifyTableProcedure.class);
48
49 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
50
51 private static long nonceGroup = HConstants.NO_NONCE;
52 private static long nonce = HConstants.NO_NONCE;
53
54 private static void setupConf(Configuration conf) {
55 conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
56 }
57
58 @BeforeClass
59 public static void setupCluster() throws Exception {
60 setupConf(UTIL.getConfiguration());
61 UTIL.startMiniCluster(1);
62 }
63
64 @AfterClass
65 public static void cleanupTest() throws Exception {
66 try {
67 UTIL.shutdownMiniCluster();
68 } catch (Exception e) {
69 LOG.warn("failure shutting down cluster", e);
70 }
71 }
72
73 @Before
74 public void setup() throws Exception {
75 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
76 nonceGroup =
77 MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
78 nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
79 }
80
81 @After
82 public void tearDown() throws Exception {
83 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
84 for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
85 LOG.info("Tear down, remove table=" + htd.getTableName());
86 UTIL.deleteTable(htd.getTableName());
87 }
88 }
89
90 @Test(timeout=60000)
91 public void testModifyTable() throws Exception {
92 final TableName tableName = TableName.valueOf("testModifyTable");
93 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
94
95 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
96 UTIL.getHBaseAdmin().disableTable(tableName);
97
98
99 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
100
101
102 long newMaxFileSize = htd.getMaxFileSize() * 2;
103 htd.setMaxFileSize(newMaxFileSize);
104 htd.setRegionReplication(3);
105
106 long procId1 = ProcedureTestingUtility.submitAndWait(
107 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
108 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
109
110 HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
111 assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
112
113
114 boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
115 long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
116 htd.setReadOnly(newReadOnlyOption);
117 htd.setMemStoreFlushSize(newMemStoreFlushSize);
118
119 long procId2 = ProcedureTestingUtility.submitAndWait(
120 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
121 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
122
123 currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
124 assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
125 assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
126 }
127
128 @Test(timeout = 60000)
129 public void testModifyTableAddCF() throws Exception {
130 final TableName tableName = TableName.valueOf("testModifyTableAddCF");
131 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
132
133 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
134 HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
135 assertEquals(1, currentHtd.getFamiliesKeys().size());
136
137
138 String cf2 = "cf2";
139 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
140 htd.addFamily(new HColumnDescriptor(cf2));
141
142 long procId = ProcedureTestingUtility.submitAndWait(
143 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
144 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
145
146 currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
147 assertEquals(2, currentHtd.getFamiliesKeys().size());
148 assertTrue(currentHtd.hasFamily(cf2.getBytes()));
149
150
151 UTIL.getHBaseAdmin().disableTable(tableName);
152 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
153 String cf3 = "cf3";
154 HTableDescriptor htd2 =
155 new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
156 htd2.addFamily(new HColumnDescriptor(cf3));
157
158 long procId2 =
159 ProcedureTestingUtility.submitAndWait(procExec,
160 new ModifyTableProcedure(procExec.getEnvironment(), htd2));
161 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
162
163 currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
164 assertTrue(currentHtd.hasFamily(cf3.getBytes()));
165 assertEquals(3, currentHtd.getFamiliesKeys().size());
166 }
167
168 @Test(timeout = 60000)
169 public void testModifyTableDeleteCF() throws Exception {
170 final TableName tableName = TableName.valueOf("testModifyTableAddCF");
171 final String cf2 = "cf2";
172 final String cf3 = "cf3";
173 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
174
175 MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf2, cf3);
176 HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
177 assertEquals(3, currentHtd.getFamiliesKeys().size());
178
179
180 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
181 htd.removeFamily(cf2.getBytes());
182
183 long procId = ProcedureTestingUtility.submitAndWait(
184 procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
185 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
186
187 currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
188 assertEquals(2, currentHtd.getFamiliesKeys().size());
189 assertFalse(currentHtd.hasFamily(cf2.getBytes()));
190
191
192 UTIL.getHBaseAdmin().disableTable(tableName);
193 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
194
195 HTableDescriptor htd2 =
196 new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
197 htd2.removeFamily(cf3.getBytes());
198
199 long procId2 =
200 ProcedureTestingUtility.submitAndWait(procExec,
201 new ModifyTableProcedure(procExec.getEnvironment(), htd2));
202 ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
203
204 currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
205 assertEquals(1, currentHtd.getFamiliesKeys().size());
206 assertFalse(currentHtd.hasFamily(cf3.getBytes()));
207 }
208
209 @Test(timeout=60000)
210 public void testRecoveryAndDoubleExecutionOffline() throws Exception {
211 final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
212 final String cf2 = "cf2";
213 final String cf3 = "cf3";
214 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
215
216
217 HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
218 procExec, tableName, null, "cf1", cf3);
219 UTIL.getHBaseAdmin().disableTable(tableName);
220
221 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
222 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
223
224
225 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
226 boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
227 htd.setCompactionEnabled(newCompactionEnableOption);
228 htd.addFamily(new HColumnDescriptor(cf2));
229 htd.removeFamily(cf3.getBytes());
230 htd.setRegionReplication(3);
231
232
233 long procId = procExec.submitProcedure(
234 new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
235
236
237 int numberOfSteps = ModifyTableState.values().length;
238 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
239 procExec,
240 procId,
241 numberOfSteps,
242 ModifyTableState.values());
243
244
245 HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
246 assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
247 assertEquals(2, currentHtd.getFamiliesKeys().size());
248
249
250 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
251 tableName, regions, false, "cf1", cf2);
252 }
253
254 @Test(timeout = 60000)
255 public void testRecoveryAndDoubleExecutionOnline() throws Exception {
256 final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
257 final String cf2 = "cf2";
258 final String cf3 = "cf3";
259 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
260
261
262 HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
263 procExec, tableName, null, "cf1", cf3);
264
265 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
266
267
268 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
269 boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
270 htd.setCompactionEnabled(newCompactionEnableOption);
271 htd.addFamily(new HColumnDescriptor(cf2));
272 htd.removeFamily(cf3.getBytes());
273
274
275 long procId = procExec.submitProcedure(
276 new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
277
278
279 int numberOfSteps = ModifyTableState.values().length;
280 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
281 ModifyTableState.values());
282
283
284 HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
285 assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
286 assertEquals(2, currentHtd.getFamiliesKeys().size());
287 assertTrue(currentHtd.hasFamily(cf2.getBytes()));
288 assertFalse(currentHtd.hasFamily(cf3.getBytes()));
289
290
291 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
292 tableName, regions, "cf1", cf2);
293 }
294
295 @Test(timeout = 60000)
296 public void testRollbackAndDoubleExecutionOnline() throws Exception {
297 final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
298 final String familyName = "cf2";
299 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
300
301
302 HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
303 procExec, tableName, null, "cf1");
304
305 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
306
307 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
308 boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
309 htd.setCompactionEnabled(newCompactionEnableOption);
310 htd.addFamily(new HColumnDescriptor(familyName));
311
312
313 long procId = procExec.submitProcedure(
314 new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
315
316
317 int numberOfSteps = ModifyTableState.values().length - 4;
318 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
319 procExec,
320 procId,
321 numberOfSteps,
322 ModifyTableState.values());
323
324
325 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
326 tableName, regions, "cf1");
327 }
328
329 @Test(timeout = 60000)
330 public void testRollbackAndDoubleExecutionOffline() throws Exception {
331 final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
332 final String familyName = "cf2";
333 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
334
335
336 HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
337 procExec, tableName, null, "cf1");
338 UTIL.getHBaseAdmin().disableTable(tableName);
339
340 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
341 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
342
343 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
344 boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
345 htd.setCompactionEnabled(newCompactionEnableOption);
346 htd.addFamily(new HColumnDescriptor(familyName));
347 htd.setRegionReplication(3);
348
349
350 long procId = procExec.submitProcedure(
351 new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
352
353
354 int numberOfSteps = ModifyTableState.values().length - 4;
355 MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
356 procExec,
357 procId,
358 numberOfSteps,
359 ModifyTableState.values());
360
361
362 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
363 tableName, regions, "cf1");
364 }
365
366 @Test(timeout = 60000)
367 public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
368 final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
369 final String familyToAddName = "cf2";
370 final String familyToRemove = "cf1";
371 final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
372
373
374 HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
375 procExec, tableName, null, familyToRemove);
376 UTIL.getHBaseAdmin().disableTable(tableName);
377
378 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
379 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
380
381 HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
382 htd.setCompactionEnabled(!htd.isCompactionEnabled());
383 htd.addFamily(new HColumnDescriptor(familyToAddName));
384 htd.removeFamily(familyToRemove.getBytes());
385 htd.setRegionReplication(3);
386
387
388 long procId = procExec.submitProcedure(
389 new ModifyTableProcedure(procExec.getEnvironment(), htd), nonceGroup, nonce);
390
391
392
393
394 int numberOfSteps = 5;
395 MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
396 procExec,
397 procId,
398 numberOfSteps,
399 ModifyTableState.values());
400
401
402 MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
403 tableName, regions, false, familyToAddName);
404 }
405
406 private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
407 return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
408 }
409 }