1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.migration;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.File;
24 import java.io.IOException;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.FileStatus;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.FsShell;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.io.FileLink;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
37 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
38 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
39 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State;
40 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.FSUtils;
43 import org.apache.hadoop.hbase.util.HFileV1Detector;
44 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
45 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46 import org.apache.hadoop.util.ToolRunner;
47 import org.apache.zookeeper.KeeperException;
48 import org.junit.AfterClass;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53 import com.google.protobuf.InvalidProtocolBufferException;
54
55
56
57
58
59
60 @Category(MediumTests.class)
61 public class TestUpgradeTo96 {
62
63 static final Log LOG = LogFactory.getLog(TestUpgradeTo96.class);
64 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65
66
67
68
69 private static FileSystem fs;
70
71
72
73 private static Path hbaseRootDir;
74 private static ZooKeeperWatcher zkw;
75
76
77
78 private static String replicationPeerZnode;
79
80
81
82 private static String tableAZnode;
83 private static ReplicationPeer peer1;
84
85
86
87 private static String peer1Znode;
88
89 @BeforeClass
90 public static void setUpBeforeClass() throws Exception {
91
92
93 TEST_UTIL.startMiniZKCluster();
94 TEST_UTIL.startMiniDFSCluster(1);
95
96 hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
97 fs = FileSystem.get(TEST_UTIL.getConfiguration());
98 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), hbaseRootDir);
99 zkw = TEST_UTIL.getZooKeeperWatcher();
100
101 Path testdir = TEST_UTIL.getDataTestDir("TestUpgradeTo96");
102
103
104 set94FSLayout(testdir);
105 setUp94Znodes();
106 }
107
108
109
110
111
112
113
114 private static void set94FSLayout(Path testdir) throws IOException, Exception {
115 File untar = TestNamespaceUpgrade.untar(new File(testdir.toString()));
116 if (!fs.exists(hbaseRootDir.getParent())) {
117
118 fs.mkdirs(hbaseRootDir.getParent());
119 }
120 FsShell shell = new FsShell(TEST_UTIL.getConfiguration());
121 shell.run(new String[] { "-put", untar.toURI().toString(), hbaseRootDir.toString() });
122
123 shell.run(new String[] { "-lsr", "/" });
124 }
125
126
127
128
129
130
131 private static void setUp94Znodes() throws IOException, KeeperException {
132
133 String rootRegionServerZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
134 ZKUtil.createWithParents(zkw, rootRegionServerZnode);
135 ZKUtil.createWithParents(zkw, zkw.backupMasterAddressesZNode);
136
137 tableAZnode = ZKUtil.joinZNode(zkw.tableZNode, "a");
138 ZKUtil.createWithParents(zkw, tableAZnode,
139 Bytes.toBytes(ZooKeeperProtos.Table.State.ENABLED.toString()));
140
141 String replicationZnode = ZKUtil.joinZNode(zkw.baseZNode, "replication");
142 replicationPeerZnode = ZKUtil.joinZNode(replicationZnode, "peers");
143 peer1Znode = ZKUtil.joinZNode(replicationPeerZnode, "1");
144 peer1 = ReplicationPeer.newBuilder().setClusterkey("abc:123:/hbase").build();
145 ZKUtil.createWithParents(zkw, peer1Znode, Bytes.toBytes(peer1.getClusterkey()));
146 }
147
148
149
150
151
152 @Test
153 public void testHFileV1Detector() throws Exception {
154 assertEquals(0, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
155 }
156
157
158
159
160
161 @Test
162 public void testHFileV1DetectorWithCorruptFiles() throws Exception {
163
164 Path tablePath = new Path(hbaseRootDir, "foo");
165 FileStatus[] regionsDir = fs.listStatus(tablePath);
166 if (regionsDir == null) throw new IOException("No Regions found for table " + "foo");
167 Path columnFamilyDir = null;
168 Path targetRegion = null;
169 for (FileStatus s : regionsDir) {
170 if (fs.exists(new Path(s.getPath(), HRegionFileSystem.REGION_INFO_FILE))) {
171 targetRegion = s.getPath();
172 break;
173 }
174 }
175 FileStatus[] cfs = fs.listStatus(targetRegion);
176 for (FileStatus f : cfs) {
177 if (f.isDirectory()) {
178 columnFamilyDir = f.getPath();
179 break;
180 }
181 }
182 LOG.debug("target columnFamilyDir: " + columnFamilyDir);
183
184 Path corruptFile = new Path(columnFamilyDir, "corrupt_file");
185 if (!fs.createNewFile(corruptFile)) throw new IOException("Couldn't create corrupt file: "
186 + corruptFile);
187 assertEquals(1, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
188
189 FileSystem.get(TEST_UTIL.getConfiguration()).delete(corruptFile, false);
190 }
191
192 @Test
193 public void testHFileLink() throws Exception {
194
195 Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
196 Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48");
197 Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48");
198 Path preNamespaceArchivePath =
199 new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48");
200 Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48");
201 boolean preNSTablePathExists = false;
202 boolean preNSArchivePathExists = false;
203 boolean preNSTempPathExists = false;
204 assertTrue(HFileLink.isHFileLink(aFileLink));
205 HFileLink hFileLink =
206 HFileLink.buildFromHFileLinkPattern(TEST_UTIL.getConfiguration(), aFileLink);
207 assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));
208
209 HFileV1Detector t = new HFileV1Detector();
210 t.setConf(TEST_UTIL.getConfiguration());
211 FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
212
213 assertTrue(fileLink.getLocations().length == 6);
214 for (Path p : fileLink.getLocations()) {
215 if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
216 if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
217 if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
218 }
219 assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);
220 }
221
222 @Test
223 public void testADirForHFileV1() throws Exception {
224 Path tablePath = new Path(hbaseRootDir, "foo");
225 System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs));
226 System.out.println("Passed: " + hbaseRootDir + "/foo");
227 assertEquals(0,
228 ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), new String[] { "-p"
229 + "foo" }));
230 }
231
232 @Test
233 public void testZnodeMigration() throws Exception {
234 String rootRSZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
235 assertTrue(ZKUtil.checkExists(zkw, rootRSZnode) > -1);
236 ToolRunner.run(TEST_UTIL.getConfiguration(), new UpgradeTo96(), new String[] { "-execute" });
237 assertEquals(-1, ZKUtil.checkExists(zkw, rootRSZnode));
238 byte[] data = ZKUtil.getData(zkw, tableAZnode);
239 assertTrue(ProtobufUtil.isPBMagicPrefix(data));
240 checkTableState(data, ZooKeeperProtos.Table.State.ENABLED);
241
242 data = ZKUtil.getData(zkw, peer1Znode);
243 assertTrue(ProtobufUtil.isPBMagicPrefix(data));
244 checkReplicationPeerData(data, peer1);
245 }
246
247 private void checkTableState(byte[] data, State expectedState)
248 throws InvalidProtocolBufferException {
249 ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder();
250 int magicLen = ProtobufUtil.lengthOfPBMagic();
251 ZooKeeperProtos.Table t = builder.mergeFrom(data, magicLen, data.length - magicLen).build();
252 assertTrue(t.getState() == expectedState);
253 }
254
255 private void checkReplicationPeerData(byte[] data, ReplicationPeer peer)
256 throws InvalidProtocolBufferException {
257 int magicLen = ProtobufUtil.lengthOfPBMagic();
258 ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
259 assertEquals(builder.mergeFrom(data, magicLen, data.length - magicLen).build().getClusterkey(),
260 peer.getClusterkey());
261
262 }
263
264 @AfterClass
265 public static void tearDownAfterClass() throws Exception {
266 TEST_UTIL.shutdownMiniHBaseCluster();
267 TEST_UTIL.shutdownMiniDFSCluster();
268 TEST_UTIL.shutdownMiniZKCluster();
269 }
270
271 }