View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.util.Comparator;
24  import java.util.HashSet;
25  import java.util.Map;
26  import java.util.TreeMap;
27  import java.util.concurrent.atomic.AtomicBoolean;
28  import java.util.concurrent.atomic.AtomicInteger;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.MetaTableAccessor;
39  import org.apache.hadoop.hbase.ScheduledChore;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.backup.HFileArchiver;
43  import org.apache.hadoop.hbase.classification.InterfaceAudience;
44  import org.apache.hadoop.hbase.client.Connection;
45  import org.apache.hadoop.hbase.client.MetaScanner;
46  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
47  import org.apache.hadoop.hbase.client.Result;
48  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.Pair;
52  import org.apache.hadoop.hbase.util.PairOfSameType;
53  import org.apache.hadoop.hbase.util.Triple;
54  
55  /**
56   * A janitor for the catalog tables.  Scans the <code>hbase:meta</code> catalog
57   * table on a period looking for unused regions to garbage collect.
58   */
59  @InterfaceAudience.Private
60  public class CatalogJanitor extends ScheduledChore {
61    private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
62    private final Server server;
63    private final MasterServices services;
64    private AtomicBoolean enabled = new AtomicBoolean(true);
65    private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
66    private final Connection connection;
67  
68    CatalogJanitor(final Server server, final MasterServices services) {
69      super("CatalogJanitor-" + server.getServerName().toShortString(), server, server
70          .getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
71      this.server = server;
72      this.services = services;
73      this.connection = server.getConnection();
74    }
75  
76    @Override
77    protected boolean initialChore() {
78      try {
79        if (this.enabled.get()) scan();
80      } catch (IOException e) {
81        LOG.warn("Failed initial scan of catalog table", e);
82        return false;
83      }
84      return true;
85    }
86  
87    /**
88     * @param enabled
89     */
90    public boolean setEnabled(final boolean enabled) {
91      return this.enabled.getAndSet(enabled);
92    }
93  
94    boolean getEnabled() {
95      return this.enabled.get();
96    }
97  
98    @Override
99    protected void chore() {
100     try {
101       AssignmentManager am = this.services.getAssignmentManager();
102       if (this.enabled.get()
103           && am != null
104           && am.isFailoverCleanupDone()
105           && am.getRegionStates().getRegionsInTransition().size() == 0) {
106         scan();
107       } else {
108         LOG.warn("CatalogJanitor disabled! Not running scan.");
109       }
110     } catch (IOException e) {
111       LOG.warn("Failed scan of catalog table", e);
112     }
113   }
114 
115   /**
116    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
117    * regions, and an ordered map of split parents.
118    * @return triple of scanned rows, map of merged regions and map of split
119    *         parent regioninfos
120    * @throws IOException
121    */
122   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents()
123       throws IOException {
124     return getMergedRegionsAndSplitParents(null);
125   }
126 
127   /**
128    * Scans hbase:meta and returns a number of scanned rows, and a map of merged
129    * regions, and an ordered map of split parents. if the given table name is
130    * null, return merged regions and split parents of all tables, else only the
131    * specified table
132    * @param tableName null represents all tables
133    * @return triple of scanned rows, and map of merged regions, and map of split
134    *         parent regioninfos
135    * @throws IOException
136    */
137   Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents(
138       final TableName tableName) throws IOException {
139     final boolean isTableSpecified = (tableName != null);
140     // TODO: Only works with single hbase:meta region currently.  Fix.
141     final AtomicInteger count = new AtomicInteger(0);
142     // Keep Map of found split parents.  There are candidates for cleanup.
143     // Use a comparator that has split parents come before its daughters.
144     final Map<HRegionInfo, Result> splitParents =
145       new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
146     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
147     // This visitor collects split parents and counts rows in the hbase:meta table
148 
149     MetaScannerVisitor visitor = new MetaScanner.MetaScannerVisitorBase() {
150       @Override
151       public boolean processRow(Result r) throws IOException {
152         if (r == null || r.isEmpty()) return true;
153         count.incrementAndGet();
154         HRegionInfo info = HRegionInfo.getHRegionInfo(r);
155         if (info == null) return true; // Keep scanning
156         if (isTableSpecified
157             && info.getTable().compareTo(tableName) > 0) {
158           // Another table, stop scanning
159           return false;
160         }
161         if (info.isSplitParent()) splitParents.put(info, r);
162         if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
163           mergedRegions.put(info, r);
164         }
165         // Returning true means "keep scanning"
166         return true;
167       }
168     };
169 
170     // Run full scan of hbase:meta catalog table passing in our custom visitor with
171     // the start row
172     MetaScanner.metaScan(this.connection, visitor, tableName);
173 
174     return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
175         count.get(), mergedRegions, splitParents);
176   }
177 
178   /**
179    * If merged region no longer holds reference to the merge regions, archive
180    * merge region on hdfs and perform deleting references in hbase:meta
181    * @param mergedRegion
182    * @param regionA
183    * @param regionB
184    * @return true if we delete references in merged region on hbase:meta and archive
185    *         the files on the file system
186    * @throws IOException
187    */
188   boolean cleanMergeRegion(final HRegionInfo mergedRegion,
189       final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
190     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
191     Path rootdir = this.services.getMasterFileSystem().getRootDir();
192     Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
193     HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
194     HRegionFileSystem regionFs = null;
195     try {
196       regionFs = HRegionFileSystem.openRegionFromFileSystem(
197           this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
198     } catch (IOException e) {
199       LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
200     }
201     if (regionFs == null || !regionFs.hasReferences(htd)) {
202       LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
203           + regionB.getRegionNameAsString()
204           + " from fs because merged region no longer holds references");
205       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
206       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
207       MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
208         mergedRegion);
209       return true;
210     }
211     return false;
212   }
213 
214   /**
215    * Run janitorial scan of catalog <code>hbase:meta</code> table looking for
216    * garbage to collect.
217    * @return number of cleaned regions
218    * @throws IOException
219    */
220   int scan() throws IOException {
221     try {
222       if (!alreadyRunning.compareAndSet(false, true)) {
223         return 0;
224       }
225       Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
226         getMergedRegionsAndSplitParents();
227       int count = scanTriple.getFirst();
228       /**
229        * clean merge regions first
230        */
231       int mergeCleaned = 0;
232       Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
233       for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
234         HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(),
235             HConstants.MERGEA_QUALIFIER);
236         HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(),
237             HConstants.MERGEB_QUALIFIER);
238         if (regionA == null || regionB == null) {
239           LOG.warn("Unexpected references regionA="
240               + (regionA == null ? "null" : regionA.getRegionNameAsString())
241               + ",regionB="
242               + (regionB == null ? "null" : regionB.getRegionNameAsString())
243               + " in merged region " + e.getKey().getRegionNameAsString());
244         } else {
245           if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
246             mergeCleaned++;
247           }
248         }
249       }
250       /**
251        * clean split parents
252        */
253       Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
254 
255       // Now work on our list of found parents. See if any we can clean up.
256       int splitCleaned = 0;
257       // regions whose parents are still around
258       HashSet<String> parentNotCleaned = new HashSet<String>();
259       for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
260         if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
261             cleanParent(e.getKey(), e.getValue())) {
262           splitCleaned++;
263         } else {
264           // We could not clean the parent, so it's daughters should not be cleaned either (HBASE-6160)
265           PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue());
266           parentNotCleaned.add(daughters.getFirst().getEncodedName());
267           parentNotCleaned.add(daughters.getSecond().getEncodedName());
268         }
269       }
270       if ((mergeCleaned + splitCleaned) != 0) {
271         LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
272             + " unreferenced merged region(s) and " + splitCleaned
273             + " unreferenced parent region(s)");
274       } else if (LOG.isTraceEnabled()) {
275         LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
276             + " unreferenced merged region(s) and " + splitCleaned
277             + " unreferenced parent region(s)");
278       }
279       return mergeCleaned + splitCleaned;
280     } finally {
281       alreadyRunning.set(false);
282     }
283   }
284 
285   /**
286    * Compare HRegionInfos in a way that has split parents sort BEFORE their
287    * daughters.
288    */
289   static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
290     Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
291     @Override
292     public int compare(HRegionInfo left, HRegionInfo right) {
293       // This comparator differs from the one HRegionInfo in that it sorts
294       // parent before daughters.
295       if (left == null) return -1;
296       if (right == null) return 1;
297       // Same table name.
298       int result = left.getTable().compareTo(right.getTable());
299       if (result != 0) return result;
300       // Compare start keys.
301       result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
302       if (result != 0) return result;
303       // Compare end keys, but flip the operands so parent comes first
304       result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
305 
306       return result;
307     }
308   }
309 
310   /**
311    * If daughters no longer hold reference to the parents, delete the parent.
312    * @param parent HRegionInfo of split offlined parent
313    * @param rowContent Content of <code>parent</code> row in
314    * <code>metaRegionName</code>
315    * @return True if we removed <code>parent</code> from meta table and from
316    * the filesystem.
317    * @throws IOException
318    */
319   boolean cleanParent(final HRegionInfo parent, Result rowContent)
320   throws IOException {
321     boolean result = false;
322     // Check whether it is a merged region and not clean reference
323     // No necessary to check MERGEB_QUALIFIER because these two qualifiers will
324     // be inserted/deleted together
325     if (rowContent.getValue(HConstants.CATALOG_FAMILY,
326         HConstants.MERGEA_QUALIFIER) != null) {
327       // wait cleaning merge region first
328       return result;
329     }
330     // Run checks on each daughter split.
331     PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent);
332     Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
333     Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
334     if (hasNoReferences(a) && hasNoReferences(b)) {
335       LOG.debug("Deleting region " + parent.getRegionNameAsString() +
336         " because daughter splits no longer hold references");
337       FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
338       if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
339       HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
340       MetaTableAccessor.deleteRegion(this.connection, parent);
341       result = true;
342     }
343     return result;
344   }
345 
346   /**
347    * @param p A pair where the first boolean says whether or not the daughter
348    * region directory exists in the filesystem and then the second boolean says
349    * whether the daughter has references to the parent.
350    * @return True the passed <code>p</code> signifies no references.
351    */
352   private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
353     return !p.getFirst() || !p.getSecond();
354   }
355 
356   /**
357    * Checks if a daughter region -- either splitA or splitB -- still holds
358    * references to parent.
359    * @param parent Parent region
360    * @param daughter Daughter region
361    * @return A pair where the first boolean says whether or not the daughter
362    * region directory exists in the filesystem and then the second boolean says
363    * whether the daughter has references to the parent.
364    * @throws IOException
365    */
366   Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
367   throws IOException {
368     if (daughter == null)  {
369       return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
370     }
371 
372     FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
373     Path rootdir = this.services.getMasterFileSystem().getRootDir();
374     Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
375 
376     Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
377 
378     HRegionFileSystem regionFs = null;
379 
380     try {
381       if (!FSUtils.isExists(fs, daughterRegionDir)) {
382         return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
383       }
384     } catch (IOException ioe) {
385       LOG.warn("Error trying to determine if daughter region exists, " +
386                "assuming exists and has references", ioe);
387       return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
388     }
389 
390     try {
391       regionFs = HRegionFileSystem.openRegionFromFileSystem(
392           this.services.getConfiguration(), fs, tabledir, daughter, true);
393     } catch (IOException e) {
394       LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
395           + ", to: " + parent.getEncodedName() + " assuming has references", e);
396       return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
397     }
398 
399     boolean references = false;
400     HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
401     for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
402       if ((references = regionFs.hasReferences(family.getNameAsString()))) {
403         break;
404       }
405     }
406     return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
407   }
408 
409   private HTableDescriptor getTableDescriptor(final TableName tableName)
410       throws FileNotFoundException, IOException {
411     return this.services.getTableDescriptors().get(tableName);
412   }
413 
414   /**
415    * Checks if the specified region has merge qualifiers, if so, try to clean
416    * them
417    * @param region
418    * @return true if the specified region doesn't have merge qualifier now
419    * @throws IOException
420    */
421   public boolean cleanMergeQualifier(final HRegionInfo region)
422       throws IOException {
423     // Get merge regions if it is a merged region and already has merge
424     // qualifier
425     Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
426         .getRegionsFromMergeQualifier(this.services.getConnection(),
427           region.getRegionName());
428     if (mergeRegions == null
429         || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
430       // It doesn't have merge qualifier, no need to clean
431       return true;
432     }
433     // It shouldn't happen, we must insert/delete these two qualifiers together
434     if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
435       LOG.error("Merged region " + region.getRegionNameAsString()
436           + " has only one merge qualifier in META.");
437       return false;
438     }
439     return cleanMergeRegion(region, mergeRegions.getFirst(),
440         mergeRegions.getSecond());
441   }
442 }