View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HTableDescriptor;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.testclassification.LargeTests;
35  import org.apache.hadoop.hbase.TableNotFoundException;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
39  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
40  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
41  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  import com.google.common.collect.Lists;
52  
53  /**
54   * Test create/using/deleting snapshots from the client
55   * <p>
56   * This is an end-to-end test for the snapshot utility
57   */
58  @Category(LargeTests.class)
59  public class TestSnapshotFromClient {
60    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
61    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
62    private static final int NUM_RS = 2;
63    private static final String STRING_TABLE_NAME = "test";
64    protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
65    protected static final TableName TABLE_NAME =
66        TableName.valueOf(STRING_TABLE_NAME);
67  
68    /**
69     * Setup the config for the cluster
70     * @throws Exception on failure
71     */
72    @BeforeClass
73    public static void setupCluster() throws Exception {
74      setupConf(UTIL.getConfiguration());
75      UTIL.startMiniCluster(NUM_RS);
76    }
77  
78    private static void setupConf(Configuration conf) {
79      // disable the ui
80      conf.setInt("hbase.regionsever.info.port", -1);
81      // change the flush size to a small amount, regulating number of store files
82      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
83      // so make sure we get a compaction when doing a load, but keep around some
84      // files in the store
85      conf.setInt("hbase.hstore.compaction.min", 10);
86      conf.setInt("hbase.hstore.compactionThreshold", 10);
87      // block writes if we get to 12 store files
88      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
89      // Enable snapshot
90      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
91      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
92        ConstantSizeRegionSplitPolicy.class.getName());
93    }
94  
95    @Before
96    public void setup() throws Exception {
97      HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
98      htd.setRegionReplication(getNumReplicas());
99      UTIL.createTable(htd, new byte[][]{TEST_FAM}, UTIL.getConfiguration());
100   }
101 
102   protected int getNumReplicas() {
103     return 1;
104   }
105 
106   @After
107   public void tearDown() throws Exception {
108     UTIL.deleteTable(TABLE_NAME);
109     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
110     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
111   }
112 
113   @AfterClass
114   public static void cleanupTest() throws Exception {
115     try {
116       UTIL.shutdownMiniCluster();
117     } catch (Exception e) {
118       LOG.warn("failure shutting down cluster", e);
119     }
120   }
121 
122   /**
123    * Test snapshotting not allowed hbase:meta and -ROOT-
124    * @throws Exception
125    */
126   @Test (timeout=300000)
127   public void testMetaTablesSnapshot() throws Exception {
128     Admin admin = UTIL.getHBaseAdmin();
129     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
130 
131     try {
132       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
133       fail("taking a snapshot of hbase:meta should not be allowed");
134     } catch (IllegalArgumentException e) {
135       // expected
136     }
137   }
138 
139   /**
140    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
141    *
142    * @throws Exception
143    */
144   @Test (timeout=300000)
145   public void testSnapshotDeletionWithRegex() throws Exception {
146     Admin admin = UTIL.getHBaseAdmin();
147     // make sure we don't fail on listing snapshots
148     SnapshotTestingUtils.assertNoSnapshots(admin);
149 
150     // put some stuff in the table
151     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
152     UTIL.loadTable(table, TEST_FAM);
153     table.close();
154 
155     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
156     admin.snapshot(snapshot1, TABLE_NAME);
157     LOG.debug("Snapshot1 completed.");
158 
159     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
160     admin.snapshot(snapshot2, TABLE_NAME);
161     LOG.debug("Snapshot2 completed.");
162 
163     String snapshot3 = "3rdTableSnapshot";
164     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
165     LOG.debug(snapshot3 + " completed.");
166 
167     // delete the first two snapshots
168     admin.deleteSnapshots("TableSnapshot.*");
169     List<SnapshotDescription> snapshots = admin.listSnapshots();
170     assertEquals(1, snapshots.size());
171     assertEquals(snapshots.get(0).getName(), snapshot3);
172 
173     admin.deleteSnapshot(snapshot3);
174     admin.close();
175   }
176   /**
177    * Test snapshotting a table that is offline
178    * @throws Exception
179    */
180   @Test (timeout=300000)
181   public void testOfflineTableSnapshot() throws Exception {
182     Admin admin = UTIL.getHBaseAdmin();
183     // make sure we don't fail on listing snapshots
184     SnapshotTestingUtils.assertNoSnapshots(admin);
185 
186     // put some stuff in the table
187     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
188     UTIL.loadTable(table, TEST_FAM, false);
189 
190     LOG.debug("FS state before disable:");
191     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
192       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
193     // XXX if this is flakey, might want to consider using the async version and looping as
194     // disableTable can succeed and still timeout.
195     admin.disableTable(TABLE_NAME);
196 
197     LOG.debug("FS state before snapshot:");
198     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
199       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
200 
201     // take a snapshot of the disabled table
202     final String SNAPSHOT_NAME = "offlineTableSnapshot";
203     byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
204 
205     SnapshotDescription desc = SnapshotDescription.newBuilder()
206       .setType(SnapshotDescription.Type.DISABLED)
207       .setTable(STRING_TABLE_NAME)
208       .setName(SNAPSHOT_NAME)
209       .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
210       .build();
211     admin.snapshot(desc);
212     LOG.debug("Snapshot completed.");
213 
214     // make sure we have the snapshot
215     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
216       snapshot, TABLE_NAME);
217 
218     // make sure its a valid snapshot
219     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
220     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
221     LOG.debug("FS state after snapshot:");
222     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
223       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
224 
225     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
226       admin, fs);
227 
228     admin.deleteSnapshot(snapshot);
229     snapshots = admin.listSnapshots();
230     SnapshotTestingUtils.assertNoSnapshots(admin);
231   }
232 
233   @Test (timeout=300000)
234   public void testSnapshotFailsOnNonExistantTable() throws Exception {
235     Admin admin = UTIL.getHBaseAdmin();
236     // make sure we don't fail on listing snapshots
237     SnapshotTestingUtils.assertNoSnapshots(admin);
238     String tableName = "_not_a_table";
239 
240     // make sure the table doesn't exist
241     boolean fail = false;
242     do {
243     try {
244       admin.getTableDescriptor(TableName.valueOf(tableName));
245       fail = true;
246           LOG.error("Table:" + tableName + " already exists, checking a new name");
247       tableName = tableName+"!";
248     } catch (TableNotFoundException e) {
249       fail = false;
250       }
251     } while (fail);
252 
253     // snapshot the non-existant table
254     try {
255       admin.snapshot("fail", TableName.valueOf(tableName));
256       fail("Snapshot succeeded even though there is not table.");
257     } catch (SnapshotCreationException e) {
258       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
259     }
260   }
261 
262   @Test (timeout=300000)
263   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
264     // test with an empty table with one region
265 
266     Admin admin = UTIL.getHBaseAdmin();
267     // make sure we don't fail on listing snapshots
268     SnapshotTestingUtils.assertNoSnapshots(admin);
269 
270     LOG.debug("FS state before disable:");
271     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
272       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
273     admin.disableTable(TABLE_NAME);
274 
275     LOG.debug("FS state before snapshot:");
276     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
277       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
278 
279     // take a snapshot of the disabled table
280     byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
281     admin.snapshot(snapshot, TABLE_NAME);
282     LOG.debug("Snapshot completed.");
283 
284     // make sure we have the snapshot
285     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
286       snapshot, TABLE_NAME);
287 
288     // make sure its a valid snapshot
289     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
290     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
291     LOG.debug("FS state after snapshot:");
292     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
293       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
294 
295     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
296     List<byte[]> nonEmptyCfs = Lists.newArrayList();
297     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
298       rootDir, admin, fs);
299 
300     admin.deleteSnapshot(snapshot);
301     snapshots = admin.listSnapshots();
302     SnapshotTestingUtils.assertNoSnapshots(admin);
303   }
304 }