View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HColumnDescriptor;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
31  import org.apache.hadoop.hbase.ProcedureInfo;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
34  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
35  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.AddColumnFamilyState;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestAddColumnFamilyProcedure {
46    private static final Log LOG = LogFactory.getLog(TestAddColumnFamilyProcedure.class);
47  
48    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49  
50    private static long nonceGroup = HConstants.NO_NONCE;
51    private static long nonce = HConstants.NO_NONCE;
52  
53    private static void setupConf(Configuration conf) {
54      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
55    }
56  
57    @BeforeClass
58    public static void setupCluster() throws Exception {
59      setupConf(UTIL.getConfiguration());
60      UTIL.startMiniCluster(1);
61    }
62  
63    @AfterClass
64    public static void cleanupTest() throws Exception {
65      try {
66        UTIL.shutdownMiniCluster();
67      } catch (Exception e) {
68        LOG.warn("failure shutting down cluster", e);
69      }
70    }
71  
72    @Before
73    public void setup() throws Exception {
74      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
75      nonceGroup =
76          MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
77      nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
78    }
79  
80    @After
81    public void tearDown() throws Exception {
82      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
83      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
84        LOG.info("Tear down, remove table=" + htd.getTableName());
85        UTIL.deleteTable(htd.getTableName());
86      }
87    }
88  
89    @Test(timeout = 60000)
90    public void testAddColumnFamily() throws Exception {
91      final TableName tableName = TableName.valueOf("testAddColumnFamily");
92      final String cf1 = "cf1";
93      final String cf2 = "cf2";
94      final HColumnDescriptor columnDescriptor1 = new HColumnDescriptor(cf1);
95      final HColumnDescriptor columnDescriptor2 = new HColumnDescriptor(cf2);
96      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
97  
98      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f3");
99  
100     // Test 1: Add a column family online
101     long procId1 = procExec.submitProcedure(
102       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor1),
103       nonceGroup,
104       nonce);
105     // Wait the completion
106     ProcedureTestingUtility.waitProcedure(procExec, procId1);
107     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
108 
109     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
110       tableName, cf1);
111 
112     // Test 2: Add a column family offline
113     UTIL.getHBaseAdmin().disableTable(tableName);
114     long procId2 = procExec.submitProcedure(
115       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor2),
116       nonceGroup + 1,
117       nonce + 1);
118     // Wait the completion
119     ProcedureTestingUtility.waitProcedure(procExec, procId2);
120     ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
121     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
122       tableName, cf2);
123   }
124 
125   @Test(timeout=60000)
126   public void testAddSameColumnFamilyTwice() throws Exception {
127     final TableName tableName = TableName.valueOf("testAddColumnFamilyTwice");
128     final String cf2 = "cf2";
129     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf2);
130 
131     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
132 
133     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");
134 
135     // add the column family
136     long procId1 = procExec.submitProcedure(
137       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
138       nonceGroup,
139       nonce);
140     // Wait the completion
141     ProcedureTestingUtility.waitProcedure(procExec, procId1);
142     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
143     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
144       tableName, cf2);
145 
146     // add the column family that exists
147     long procId2 = procExec.submitProcedure(
148       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
149       nonceGroup + 1,
150       nonce + 1);
151     // Wait the completion
152     ProcedureTestingUtility.waitProcedure(procExec, procId2);
153 
154     // Second add should fail with InvalidFamilyOperationException
155     ProcedureInfo result = procExec.getResult(procId2);
156     assertTrue(result.isFailed());
157     LOG.debug("Add failed with exception: " + result.getExceptionFullMessage());
158     assertTrue(
159       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
160 
161     // Do the same add the existing column family - this time offline
162     UTIL.getHBaseAdmin().disableTable(tableName);
163     long procId3 = procExec.submitProcedure(
164       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
165       nonceGroup + 2,
166       nonce + 2);
167     // Wait the completion
168     ProcedureTestingUtility.waitProcedure(procExec, procId3);
169 
170     // Second add should fail with InvalidFamilyOperationException
171     result = procExec.getResult(procId3);
172     assertTrue(result.isFailed());
173     LOG.debug("Add failed with exception: " + result.getExceptionFullMessage());
174     assertTrue(
175       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
176   }
177 
178   @Test(timeout=60000)
179   public void testAddSameColumnFamilyTwiceWithSameNonce() throws Exception {
180     final TableName tableName = TableName.valueOf("testAddSameColumnFamilyTwiceWithSameNonce");
181     final String cf2 = "cf2";
182     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf2);
183 
184     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
185 
186     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");
187 
188     // add the column family
189     long procId1 = procExec.submitProcedure(
190       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
191       nonceGroup,
192       nonce);
193     long procId2 = procExec.submitProcedure(
194       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
195       nonceGroup,
196       nonce);
197     // Wait the completion
198     ProcedureTestingUtility.waitProcedure(procExec, procId1);
199     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
200     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
201       tableName, cf2);
202 
203     // Wait the completion and expect not fail - because it is the same proc
204     ProcedureTestingUtility.waitProcedure(procExec, procId2);
205     ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
206     assertTrue(procId1 == procId2);
207   }
208 
209   @Test(timeout = 60000)
210   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
211     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
212     final String cf4 = "cf4";
213     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf4);
214     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
215     // create the table
216     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3");
217     UTIL.getHBaseAdmin().disableTable(tableName);
218 
219     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
220     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
221 
222     // Start the AddColumnFamily procedure && kill the executor
223     long procId = procExec.submitProcedure(
224       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
225       nonceGroup,
226       nonce);
227 
228     // Restart the executor and execute the step twice
229     int numberOfSteps = AddColumnFamilyState.values().length;
230     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
231       AddColumnFamilyState.values());
232 
233     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
234       tableName, cf4);
235   }
236 
237   @Test(timeout = 60000)
238   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
239     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
240     final String cf5 = "cf5";
241     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf5);
242     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
243     // create the table
244     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3");
245 
246     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
247     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
248 
249     // Start the AddColumnFamily procedure && kill the executor
250     long procId = procExec.submitProcedure(
251       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
252       nonceGroup,
253       nonce);
254 
255     // Restart the executor and execute the step twice
256     int numberOfSteps = AddColumnFamilyState.values().length;
257     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
258       AddColumnFamilyState.values());
259 
260     MasterProcedureTestingUtility.validateColumnFamilyAddition(UTIL.getHBaseCluster().getMaster(),
261       tableName, cf5);
262   }
263 
264   @Test(timeout = 60000)
265   public void testRollbackAndDoubleExecution() throws Exception {
266     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
267     final String cf6 = "cf6";
268     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf6);
269     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
270 
271     // create the table
272     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
273     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
274     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
275 
276     // Start the AddColumnFamily procedure && kill the executor
277     long procId = procExec.submitProcedure(
278       new AddColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
279       nonceGroup,
280       nonce);
281 
282     int numberOfSteps = AddColumnFamilyState.values().length - 2; // failing in the middle of proc
283     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps,
284       AddColumnFamilyState.values());
285 
286     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
287       tableName, cf6);
288   }
289 
290   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
291     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
292   }
293 }