View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.ChoreService;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.client.Admin;
42  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.HRegionServer;
46  import org.apache.hadoop.hbase.regionserver.Region;
47  import org.apache.hadoop.hbase.testclassification.MediumTests;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.FSUtils;
50  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
51  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
52  import org.apache.hadoop.hbase.util.StoppableImplementation;
53  import org.junit.After;
54  import org.junit.AfterClass;
55  import org.junit.Assert;
56  import org.junit.BeforeClass;
57  import org.junit.Test;
58  import org.junit.experimental.categories.Category;
59  
60  /**
61   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
62   * a region
63   */
64  @Category(MediumTests.class)
65  public class TestHFileArchiving {
66  
67    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
68    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
69    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
70  
71    /**
72     * Setup the config for the cluster
73     */
74    @BeforeClass
75    public static void setupCluster() throws Exception {
76      setupConf(UTIL.getConfiguration());
77      UTIL.startMiniCluster();
78  
79      // We don't want the cleaner to remove files. The tests do that.
80      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().cancel(true);
81    }
82  
83    private static void setupConf(Configuration conf) {
84      // disable the ui
85      conf.setInt("hbase.regionsever.info.port", -1);
86      // drop the memstore size so we get flushes
87      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
88      // disable major compactions
89      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
90  
91      // prevent aggressive region split
92      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93        ConstantSizeRegionSplitPolicy.class.getName());
94    }
95  
96    @After
97    public void tearDown() throws Exception {
98      // cleanup the archive directory
99      try {
100       clearArchiveDirectory();
101     } catch (IOException e) {
102       Assert.fail("Failure to delete archive directory:" + e.getMessage());
103     }
104   }
105 
106   @AfterClass
107   public static void cleanupTest() throws Exception {
108     try {
109       UTIL.shutdownMiniCluster();
110     } catch (Exception e) {
111       // NOOP;
112     }
113   }
114 
115   @Test
116   public void testRemovesRegionDirOnArchive() throws Exception {
117     TableName TABLE_NAME =
118         TableName.valueOf("testRemovesRegionDirOnArchive");
119     UTIL.createTable(TABLE_NAME, TEST_FAM);
120 
121     final Admin admin = UTIL.getHBaseAdmin();
122 
123     // get the current store files for the region
124     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
125     // make sure we only have 1 region serving this table
126     assertEquals(1, servingRegions.size());
127     HRegion region = servingRegions.get(0);
128 
129     // and load the table
130     UTIL.loadRegion(region, TEST_FAM);
131 
132     // shutdown the table so we can manipulate the files
133     admin.disableTable(TABLE_NAME);
134 
135     FileSystem fs = UTIL.getTestFileSystem();
136 
137     // now attempt to depose the region
138     Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
139     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
140 
141     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
142 
143     // check for the existence of the archive directory and some files in it
144     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
145     assertTrue(fs.exists(archiveDir));
146 
147     // check to make sure the store directory was copied
148     // check to make sure the store directory was copied
149     FileStatus[] stores = fs.listStatus(archiveDir, new PathFilter() {
150       @Override
151       public boolean accept(Path p) {
152         if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
153           return false;
154         }
155         return true;
156       }
157     });
158     assertTrue(stores.length == 1);
159 
160     // make sure we archived the store files
161     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
162     assertTrue(storeFiles.length > 0);
163 
164     // then ensure the region's directory isn't present
165     assertFalse(fs.exists(regionDir));
166 
167     UTIL.deleteTable(TABLE_NAME);
168   }
169 
170   /**
171    * Test that the region directory is removed when we archive a region without store files, but
172    * still has hidden files.
173    * @throws Exception
174    */
175   @Test
176   public void testDeleteRegionWithNoStoreFiles() throws Exception {
177     TableName TABLE_NAME =
178         TableName.valueOf("testDeleteRegionWithNoStoreFiles");
179     UTIL.createTable(TABLE_NAME, TEST_FAM);
180 
181     // get the current store files for the region
182     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
183     // make sure we only have 1 region serving this table
184     assertEquals(1, servingRegions.size());
185     HRegion region = servingRegions.get(0);
186 
187     FileSystem fs = region.getRegionFileSystem().getFileSystem();
188 
189     // make sure there are some files in the regiondir
190     Path rootDir = FSUtils.getRootDir(fs.getConf());
191     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
192     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
193     Assert.assertNotNull("No files in the region directory", regionFiles);
194     if (LOG.isDebugEnabled()) {
195       List<Path> files = new ArrayList<Path>();
196       for (FileStatus file : regionFiles) {
197         files.add(file.getPath());
198       }
199       LOG.debug("Current files:" + files);
200     }
201     // delete the visible folders so we just have hidden files/folders
202     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
203     PathFilter nonHidden = new PathFilter() {
204       @Override
205       public boolean accept(Path file) {
206         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
207       }
208     };
209     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
210     for (FileStatus store : storeDirs) {
211       LOG.debug("Deleting store for test");
212       fs.delete(store.getPath(), true);
213     }
214 
215     // then archive the region
216     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
217 
218     // and check to make sure the region directoy got deleted
219     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
220 
221     UTIL.deleteTable(TABLE_NAME);
222   }
223 
224   @Test
225   public void testArchiveOnTableDelete() throws Exception {
226     TableName TABLE_NAME =
227         TableName.valueOf("testArchiveOnTableDelete");
228     UTIL.createTable(TABLE_NAME, TEST_FAM);
229 
230     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
231     // make sure we only have 1 region serving this table
232     assertEquals(1, servingRegions.size());
233     Region region = servingRegions.get(0);
234 
235     // get the parent RS and monitor
236     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
237     FileSystem fs = hrs.getFileSystem();
238 
239     // put some data on the region
240     LOG.debug("-------Loading table");
241     UTIL.loadRegion(region, TEST_FAM);
242 
243     // get the hfiles in the region
244     List<Region> regions = hrs.getOnlineRegions(TABLE_NAME);
245     assertEquals("More that 1 region for test table.", 1, regions.size());
246 
247     region = regions.get(0);
248     // wait for all the compactions to complete
249     region.waitForFlushesAndCompactions();
250 
251     // disable table to prevent new updates
252     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
253     LOG.debug("Disabled table");
254 
255     // remove all the files from the archive to get a fair comparison
256     clearArchiveDirectory();
257 
258     // then get the current store files
259     byte[][]columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
260     List<String> storeFiles = region.getStoreFileList(columns);
261 
262     // then delete the table so the hfiles get archived
263     UTIL.deleteTable(TABLE_NAME);
264     LOG.debug("Deleted table");
265 
266     assertArchiveFiles(fs, storeFiles, 30000);
267   }
268 
269   private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
270     long end = System.currentTimeMillis() + timeout;
271     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
272     List<String> archivedFiles = new ArrayList<String>();
273 
274     // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
275     // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
276     while (System.currentTimeMillis() < end) {
277       archivedFiles = getAllFileNames(fs, archiveDir);
278       if (archivedFiles.size() >= storeFiles.size()) {
279         break;
280       }
281     }
282 
283     Collections.sort(storeFiles);
284     Collections.sort(archivedFiles);
285 
286     LOG.debug("Store files:");
287     for (int i = 0; i < storeFiles.size(); i++) {
288       LOG.debug(i + " - " + storeFiles.get(i));
289     }
290     LOG.debug("Archive files:");
291     for (int i = 0; i < archivedFiles.size(); i++) {
292       LOG.debug(i + " - " + archivedFiles.get(i));
293     }
294 
295     assertTrue("Archived files are missing some of the store files!",
296       archivedFiles.containsAll(storeFiles));
297   }
298 
299 
300   /**
301    * Test that the store files are archived when a column family is removed.
302    * @throws Exception
303    */
304   @Test
305   public void testArchiveOnTableFamilyDelete() throws Exception {
306     TableName TABLE_NAME =
307         TableName.valueOf("testArchiveOnTableFamilyDelete");
308     UTIL.createTable(TABLE_NAME, new byte[][] {TEST_FAM, Bytes.toBytes("fam2")});
309 
310     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
311     // make sure we only have 1 region serving this table
312     assertEquals(1, servingRegions.size());
313     Region region = servingRegions.get(0);
314 
315     // get the parent RS and monitor
316     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
317     FileSystem fs = hrs.getFileSystem();
318 
319     // put some data on the region
320     LOG.debug("-------Loading table");
321     UTIL.loadRegion(region, TEST_FAM);
322 
323     // get the hfiles in the region
324     List<Region> regions = hrs.getOnlineRegions(TABLE_NAME);
325     assertEquals("More that 1 region for test table.", 1, regions.size());
326 
327     region = regions.get(0);
328     // wait for all the compactions to complete
329     region.waitForFlushesAndCompactions();
330 
331     // disable table to prevent new updates
332     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
333     LOG.debug("Disabled table");
334 
335     // remove all the files from the archive to get a fair comparison
336     clearArchiveDirectory();
337 
338     // then get the current store files
339     byte[][]columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
340     List<String> storeFiles = region.getStoreFileList(columns);
341 
342     // then delete the table so the hfiles get archived
343     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
344 
345     assertArchiveFiles(fs, storeFiles, 30000);
346 
347     UTIL.deleteTable(TABLE_NAME);
348   }
349 
350   /**
351    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
352    */
353   @Test
354   public void testCleaningRace() throws Exception {
355     final long TEST_TIME = 20 * 1000;
356     final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
357 
358     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
359     Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
360     FileSystem fs = UTIL.getTestFileSystem();
361 
362     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
363     Path regionDir = new Path(FSUtils.getTableDir(new Path("./"),
364         TableName.valueOf("table")), "abcdef");
365     Path familyDir = new Path(regionDir, "cf");
366 
367     Path sourceRegionDir = new Path(rootDir, regionDir);
368     fs.mkdirs(sourceRegionDir);
369 
370     Stoppable stoppable = new StoppableImplementation();
371 
372     // The cleaner should be looping without long pauses to reproduce the race condition.
373     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
374     try {
375       choreService.scheduleChore(cleaner);
376 
377       // Keep creating/archiving new files while the cleaner is running in the other thread
378       long startTime = System.currentTimeMillis();
379       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
380         Path file = new Path(familyDir,  String.valueOf(fid));
381         Path sourceFile = new Path(rootDir, file);
382         Path archiveFile = new Path(archiveDir, file);
383 
384         fs.createNewFile(sourceFile);
385 
386         try {
387           // Try to archive the file
388           HFileArchiver.archiveRegion(fs, rootDir,
389               sourceRegionDir.getParent(), sourceRegionDir);
390 
391           // The archiver succeded, the file is no longer in the original location
392           // but it's in the archive location.
393           LOG.debug("hfile=" + fid + " should be in the archive");
394           assertTrue(fs.exists(archiveFile));
395           assertFalse(fs.exists(sourceFile));
396         } catch (IOException e) {
397           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
398           // in this case, the file should not be archived, and we should have the file
399           // in the original location.
400           LOG.debug("hfile=" + fid + " should be in the source location");
401           assertFalse(fs.exists(archiveFile));
402           assertTrue(fs.exists(sourceFile));
403 
404           // Avoid to have this file in the next run
405           fs.delete(sourceFile, false);
406         }
407       }
408     } finally {
409       stoppable.stop("test end");
410       cleaner.cancel(true);
411       choreService.shutdown();
412       fs.delete(rootDir, true);
413     }
414   }
415 
416   private void clearArchiveDirectory() throws IOException {
417     UTIL.getTestFileSystem().delete(
418       new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
419   }
420 
421   /**
422    * Get the names of all the files below the given directory
423    * @param fs
424    * @param archiveDir
425    * @return
426    * @throws IOException
427    */
428   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
429     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, new PathFilter() {
430       @Override
431       public boolean accept(Path p) {
432         if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
433           return false;
434         }
435         return true;
436       }
437     });
438     return recurseOnFiles(fs, files, new ArrayList<String>());
439   }
440 
441   /** Recursively lookup all the file names under the file[] array **/
442   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
443       throws IOException {
444     if (files == null || files.length == 0) return fileNames;
445 
446     for (FileStatus file : files) {
447       if (file.isDirectory()) {
448         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
449       } else fileNames.add(file.getPath().getName());
450     }
451     return fileNames;
452   }
453 }