1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.Comparator;
24 import java.util.HashSet;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicInteger;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.MetaTableAccessor;
39 import org.apache.hadoop.hbase.ScheduledChore;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.backup.HFileArchiver;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.client.Connection;
45 import org.apache.hadoop.hbase.client.MetaScanner;
46 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Pair;
52 import org.apache.hadoop.hbase.util.PairOfSameType;
53 import org.apache.hadoop.hbase.util.Triple;
54
55
56
57
58
59 @InterfaceAudience.Private
60 public class CatalogJanitor extends ScheduledChore {
61 private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
62 private final Server server;
63 private final MasterServices services;
64 private AtomicBoolean enabled = new AtomicBoolean(true);
65 private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
66 private final Connection connection;
67
68 CatalogJanitor(final Server server, final MasterServices services) {
69 super("CatalogJanitor-" + server.getServerName().toShortString(), server, server
70 .getConfiguration().getInt("hbase.catalogjanitor.interval", 300000));
71 this.server = server;
72 this.services = services;
73 this.connection = server.getConnection();
74 }
75
76 @Override
77 protected boolean initialChore() {
78 try {
79 if (this.enabled.get()) scan();
80 } catch (IOException e) {
81 LOG.warn("Failed initial scan of catalog table", e);
82 return false;
83 }
84 return true;
85 }
86
87
88
89
90 public boolean setEnabled(final boolean enabled) {
91 return this.enabled.getAndSet(enabled);
92 }
93
94 boolean getEnabled() {
95 return this.enabled.get();
96 }
97
98 @Override
99 protected void chore() {
100 try {
101 AssignmentManager am = this.services.getAssignmentManager();
102 if (this.enabled.get()
103 && am != null
104 && am.isFailoverCleanupDone()
105 && am.getRegionStates().getRegionsInTransition().size() == 0) {
106 scan();
107 } else {
108 LOG.warn("CatalogJanitor disabled! Not running scan.");
109 }
110 } catch (IOException e) {
111 LOG.warn("Failed scan of catalog table", e);
112 }
113 }
114
115
116
117
118
119
120
121
122 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents()
123 throws IOException {
124 return getMergedRegionsAndSplitParents(null);
125 }
126
127
128
129
130
131
132
133
134
135
136
137 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents(
138 final TableName tableName) throws IOException {
139 final boolean isTableSpecified = (tableName != null);
140
141 final AtomicInteger count = new AtomicInteger(0);
142
143
144 final Map<HRegionInfo, Result> splitParents =
145 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
146 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
147
148
149 MetaScannerVisitor visitor = new MetaScanner.MetaScannerVisitorBase() {
150 @Override
151 public boolean processRow(Result r) throws IOException {
152 if (r == null || r.isEmpty()) return true;
153 count.incrementAndGet();
154 HRegionInfo info = HRegionInfo.getHRegionInfo(r);
155 if (info == null) return true;
156 if (isTableSpecified
157 && info.getTable().compareTo(tableName) > 0) {
158
159 return false;
160 }
161 if (info.isSplitParent()) splitParents.put(info, r);
162 if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
163 mergedRegions.put(info, r);
164 }
165
166 return true;
167 }
168 };
169
170
171
172 MetaScanner.metaScan(this.connection, visitor, tableName);
173
174 return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
175 count.get(), mergedRegions, splitParents);
176 }
177
178
179
180
181
182
183
184
185
186
187
188 boolean cleanMergeRegion(final HRegionInfo mergedRegion,
189 final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
190 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
191 Path rootdir = this.services.getMasterFileSystem().getRootDir();
192 Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
193 HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
194 HRegionFileSystem regionFs = null;
195 try {
196 regionFs = HRegionFileSystem.openRegionFromFileSystem(
197 this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
198 } catch (IOException e) {
199 LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
200 }
201 if (regionFs == null || !regionFs.hasReferences(htd)) {
202 LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
203 + regionB.getRegionNameAsString()
204 + " from fs because merged region no longer holds references");
205 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
206 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
207 MetaTableAccessor.deleteMergeQualifiers(server.getConnection(),
208 mergedRegion);
209 return true;
210 }
211 return false;
212 }
213
214
215
216
217
218
219
220 int scan() throws IOException {
221 try {
222 if (!alreadyRunning.compareAndSet(false, true)) {
223 return 0;
224 }
225 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
226 getMergedRegionsAndSplitParents();
227 int count = scanTriple.getFirst();
228
229
230
231 int mergeCleaned = 0;
232 Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
233 for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
234 HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(),
235 HConstants.MERGEA_QUALIFIER);
236 HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(),
237 HConstants.MERGEB_QUALIFIER);
238 if (regionA == null || regionB == null) {
239 LOG.warn("Unexpected references regionA="
240 + (regionA == null ? "null" : regionA.getRegionNameAsString())
241 + ",regionB="
242 + (regionB == null ? "null" : regionB.getRegionNameAsString())
243 + " in merged region " + e.getKey().getRegionNameAsString());
244 } else {
245 if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
246 mergeCleaned++;
247 }
248 }
249 }
250
251
252
253 Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
254
255
256 int splitCleaned = 0;
257
258 HashSet<String> parentNotCleaned = new HashSet<String>();
259 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
260 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
261 cleanParent(e.getKey(), e.getValue())) {
262 splitCleaned++;
263 } else {
264
265 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue());
266 parentNotCleaned.add(daughters.getFirst().getEncodedName());
267 parentNotCleaned.add(daughters.getSecond().getEncodedName());
268 }
269 }
270 if ((mergeCleaned + splitCleaned) != 0) {
271 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
272 + " unreferenced merged region(s) and " + splitCleaned
273 + " unreferenced parent region(s)");
274 } else if (LOG.isTraceEnabled()) {
275 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
276 + " unreferenced merged region(s) and " + splitCleaned
277 + " unreferenced parent region(s)");
278 }
279 return mergeCleaned + splitCleaned;
280 } finally {
281 alreadyRunning.set(false);
282 }
283 }
284
285
286
287
288
289 static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
290 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
291 @Override
292 public int compare(HRegionInfo left, HRegionInfo right) {
293
294
295 if (left == null) return -1;
296 if (right == null) return 1;
297
298 int result = left.getTable().compareTo(right.getTable());
299 if (result != 0) return result;
300
301 result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
302 if (result != 0) return result;
303
304 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
305
306 return result;
307 }
308 }
309
310
311
312
313
314
315
316
317
318
319 boolean cleanParent(final HRegionInfo parent, Result rowContent)
320 throws IOException {
321 boolean result = false;
322
323
324
325 if (rowContent.getValue(HConstants.CATALOG_FAMILY,
326 HConstants.MERGEA_QUALIFIER) != null) {
327
328 return result;
329 }
330
331 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent);
332 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
333 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
334 if (hasNoReferences(a) && hasNoReferences(b)) {
335 LOG.debug("Deleting region " + parent.getRegionNameAsString() +
336 " because daughter splits no longer hold references");
337 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
338 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
339 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
340 MetaTableAccessor.deleteRegion(this.connection, parent);
341 result = true;
342 }
343 return result;
344 }
345
346
347
348
349
350
351
352 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
353 return !p.getFirst() || !p.getSecond();
354 }
355
356
357
358
359
360
361
362
363
364
365
366 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
367 throws IOException {
368 if (daughter == null) {
369 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
370 }
371
372 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
373 Path rootdir = this.services.getMasterFileSystem().getRootDir();
374 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
375
376 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
377
378 HRegionFileSystem regionFs = null;
379
380 try {
381 if (!FSUtils.isExists(fs, daughterRegionDir)) {
382 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
383 }
384 } catch (IOException ioe) {
385 LOG.warn("Error trying to determine if daughter region exists, " +
386 "assuming exists and has references", ioe);
387 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
388 }
389
390 try {
391 regionFs = HRegionFileSystem.openRegionFromFileSystem(
392 this.services.getConfiguration(), fs, tabledir, daughter, true);
393 } catch (IOException e) {
394 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
395 + ", to: " + parent.getEncodedName() + " assuming has references", e);
396 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
397 }
398
399 boolean references = false;
400 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
401 for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
402 if ((references = regionFs.hasReferences(family.getNameAsString()))) {
403 break;
404 }
405 }
406 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
407 }
408
409 private HTableDescriptor getTableDescriptor(final TableName tableName)
410 throws FileNotFoundException, IOException {
411 return this.services.getTableDescriptors().get(tableName);
412 }
413
414
415
416
417
418
419
420
421 public boolean cleanMergeQualifier(final HRegionInfo region)
422 throws IOException {
423
424
425 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaTableAccessor
426 .getRegionsFromMergeQualifier(this.services.getConnection(),
427 region.getRegionName());
428 if (mergeRegions == null
429 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
430
431 return true;
432 }
433
434 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
435 LOG.error("Merged region " + region.getRegionNameAsString()
436 + " has only one merge qualifier in META.");
437 return false;
438 }
439 return cleanMergeRegion(region, mergeRegions.getFirst(),
440 mergeRegions.getSecond());
441 }
442 }