View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.*;
25  import org.apache.hadoop.hbase.client.Admin;
26  import org.apache.hadoop.hbase.regionserver.Region;
27  import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
28  import org.apache.hadoop.hbase.testclassification.MediumTests;
29  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31  import org.apache.hadoop.hdfs.MiniDFSCluster;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.ServerLoad;
35  import org.apache.hadoop.hbase.RegionLoad;
36  
37  import java.io.*;
38  import java.util.*;
39  
40  import org.junit.*;
41  import org.junit.experimental.categories.Category;
42  
43  import static org.junit.Assert.assertEquals;
44  import static org.junit.Assert.assertNotNull;
45  import static org.junit.Assert.assertTrue;
46  import static org.junit.Assert.assertFalse;
47  
48  /**
49   * Test coprocessors class loading.
50   */
51  @Category(MediumTests.class)
52  public class TestClassLoading {
53    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    private static MiniDFSCluster cluster;
57  
58    static final TableName tableName = TableName.valueOf("TestClassLoading");
59    static final String cpName1 = "TestCP1";
60    static final String cpName2 = "TestCP2";
61    static final String cpName3 = "TestCP3";
62    static final String cpName4 = "TestCP4";
63    static final String cpName5 = "TestCP5";
64    static final String cpName6 = "TestCP6";
65  
66    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
67    // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
68    private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
69    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
70    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
71  
72    private static final String[] regionServerSystemCoprocessors =
73        new String[]{
74        regionServerCoprocessor.getSimpleName()
75    };
76  
77    private static final String[] masterRegionServerSystemCoprocessors = new String[] {
78        regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
79        regionServerCoprocessor.getSimpleName() };
80  
81    @BeforeClass
82    public static void setUpBeforeClass() throws Exception {
83      Configuration conf = TEST_UTIL.getConfiguration();
84  
85      // regionCoprocessor1 will be loaded on all regionservers, since it is
86      // loaded for any tables (user or meta).
87      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
88          regionCoprocessor1.getName());
89  
90      // regionCoprocessor2 will be loaded only on regionservers that serve a
91      // user table region. Therefore, if there are no user tables loaded,
92      // this coprocessor will not be loaded on any regionserver.
93      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
94          regionCoprocessor2.getName());
95  
96      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
97          regionServerCoprocessor.getName());
98      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
99          masterCoprocessor.getName());
100     TEST_UTIL.startMiniCluster(1);
101     cluster = TEST_UTIL.getDFSCluster();
102   }
103 
104   @AfterClass
105   public static void tearDownAfterClass() throws Exception {
106     TEST_UTIL.shutdownMiniCluster();
107   }
108 
109   static File buildCoprocessorJar(String className) throws Exception {
110     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
111       "public class " + className + " extends BaseRegionObserver {}";
112     return ClassLoaderTestHelper.buildJar(
113       TEST_UTIL.getDataTestDir().toString(), className, code);
114   }
115 
116   @Test
117   // HBASE-3516: Test CP Class loading from HDFS
118   public void testClassLoadingFromHDFS() throws Exception {
119     FileSystem fs = cluster.getFileSystem();
120 
121     File jarFile1 = buildCoprocessorJar(cpName1);
122     File jarFile2 = buildCoprocessorJar(cpName2);
123 
124     // copy the jars into dfs
125     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
126       new Path(fs.getUri().toString() + Path.SEPARATOR));
127     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
128       jarFile1.getName();
129     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
130     assertTrue("Copy jar file to HDFS failed.",
131       fs.exists(pathOnHDFS1));
132     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
133 
134     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
135         new Path(fs.getUri().toString() + Path.SEPARATOR));
136     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
137       jarFile2.getName();
138     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
139     assertTrue("Copy jar file to HDFS failed.",
140       fs.exists(pathOnHDFS2));
141     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
142 
143     // create a table that references the coprocessors
144     HTableDescriptor htd = new HTableDescriptor(tableName);
145     htd.addFamily(new HColumnDescriptor("test"));
146       // without configuration values
147     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
148       "|" + Coprocessor.PRIORITY_USER);
149       // with configuration values
150     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
151       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
152     Admin admin = TEST_UTIL.getHBaseAdmin();
153     if (admin.tableExists(tableName)) {
154       if (admin.isTableEnabled(tableName)) {
155         admin.disableTable(tableName);
156       }
157       admin.deleteTable(tableName);
158     }
159     CoprocessorClassLoader.clearCache();
160     byte[] startKey = {10, 63};
161     byte[] endKey = {12, 43};
162     admin.createTable(htd, startKey, endKey, 4);
163     waitForTable(htd.getTableName());
164 
165     // verify that the coprocessors were loaded
166     boolean foundTableRegion=false;
167     boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
168     Map<Region, Set<ClassLoader>> regionsActiveClassLoaders =
169         new HashMap<Region, Set<ClassLoader>>();
170     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
171     for (Region region:
172         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
173       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
174         foundTableRegion = true;
175         CoprocessorEnvironment env;
176         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
177         found1 = found1 && (env != null);
178         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
179         found2 = found2 && (env != null);
180         if (env != null) {
181           Configuration conf = env.getConfiguration();
182           found2_k1 = found2_k1 && (conf.get("k1") != null);
183           found2_k2 = found2_k2 && (conf.get("k2") != null);
184           found2_k3 = found2_k3 && (conf.get("k3") != null);
185         } else {
186           found2_k1 = found2_k2 = found2_k3 = false;
187         }
188         regionsActiveClassLoaders
189             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
190       }
191     }
192 
193     assertTrue("No region was found for table " + tableName, foundTableRegion);
194     assertTrue("Class " + cpName1 + " was missing on a region", found1);
195     assertTrue("Class " + cpName2 + " was missing on a region", found2);
196     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
197     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
198     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
199     // check if CP classloaders are cached
200     assertNotNull(jarFileOnHDFS1 + " was not cached",
201       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
202     assertNotNull(jarFileOnHDFS2 + " was not cached",
203       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
204     //two external jar used, should be one classloader per jar
205     assertEquals("The number of cached classloaders should be equal to the number" +
206       " of external jar files",
207       2, CoprocessorClassLoader.getAllCached().size());
208     //check if region active classloaders are shared across all RS regions
209     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
210       CoprocessorClassLoader.getAllCached());
211     for (Map.Entry<Region, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
212       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
213         + " ClassLoader Cache:" + externalClassLoaders
214         + " Region ClassLoaders:" + regionCP.getValue(),
215         externalClassLoaders.containsAll(regionCP.getValue()));
216     }
217   }
218 
219   private String getLocalPath(File file) {
220     return new Path(file.toURI()).toString();
221   }
222 
223   @Test
224   // HBASE-3516: Test CP Class loading from local file system
225   public void testClassLoadingFromLocalFS() throws Exception {
226     File jarFile = buildCoprocessorJar(cpName3);
227 
228     // create a table that references the jar
229     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
230     htd.addFamily(new HColumnDescriptor("test"));
231     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
232       Coprocessor.PRIORITY_USER);
233     Admin admin = TEST_UTIL.getHBaseAdmin();
234     admin.createTable(htd);
235     waitForTable(htd.getTableName());
236 
237     // verify that the coprocessor was loaded
238     boolean found = false;
239     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
240     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
241       if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
242         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
243       }
244     }
245     assertTrue("Class " + cpName3 + " was missing on a region", found);
246   }
247 
248   @Test
249   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
250   public void testPrivateClassLoader() throws Exception {
251     File jarFile = buildCoprocessorJar(cpName4);
252 
253     // create a table that references the jar
254     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
255     htd.addFamily(new HColumnDescriptor("test"));
256     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
257       Coprocessor.PRIORITY_USER);
258     Admin admin = TEST_UTIL.getHBaseAdmin();
259     admin.createTable(htd);
260     waitForTable(htd.getTableName());
261 
262     // verify that the coprocessor was loaded correctly
263     boolean found = false;
264     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
265     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
266       if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) {
267         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
268         if (cp != null) {
269           found = true;
270           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
271             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
272         }
273       }
274     }
275     assertTrue("Class " + cpName4 + " was missing on a region", found);
276   }
277 
278   @Test
279   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
280   // less strict
281   public void testHBase3810() throws Exception {
282     // allowed value pattern: [path] | class name | [priority] | [key values]
283 
284     File jarFile1 = buildCoprocessorJar(cpName1);
285     File jarFile2 = buildCoprocessorJar(cpName2);
286     File jarFile5 = buildCoprocessorJar(cpName5);
287     File jarFile6 = buildCoprocessorJar(cpName6);
288 
289     String cpKey1 = "COPROCESSOR$1";
290     String cpKey2 = " Coprocessor$2 ";
291     String cpKey3 = " coprocessor$03 ";
292 
293     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
294         Coprocessor.PRIORITY_USER;
295     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
296     // load from default class loader
297     String cpValue3 =
298         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
299 
300     // create a table that references the jar
301     HTableDescriptor htd = new HTableDescriptor(tableName);
302     htd.addFamily(new HColumnDescriptor("test"));
303 
304     // add 3 coprocessors by setting htd attributes directly.
305     htd.setValue(cpKey1, cpValue1);
306     htd.setValue(cpKey2, cpValue2);
307     htd.setValue(cpKey3, cpValue3);
308 
309     // add 2 coprocessor by using new htd.addCoprocessor() api
310     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
311         Coprocessor.PRIORITY_USER, null);
312     Map<String, String> kvs = new HashMap<String, String>();
313     kvs.put("k1", "v1");
314     kvs.put("k2", "v2");
315     kvs.put("k3", "v3");
316     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
317         Coprocessor.PRIORITY_USER, kvs);
318 
319     Admin admin = TEST_UTIL.getHBaseAdmin();
320     if (admin.tableExists(tableName)) {
321       if (admin.isTableEnabled(tableName)) {
322         admin.disableTable(tableName);
323       }
324       admin.deleteTable(tableName);
325     }
326     admin.createTable(htd);
327     waitForTable(htd.getTableName());
328 
329     // verify that the coprocessor was loaded
330     boolean found_2 = false, found_1 = false, found_3 = false,
331         found_5 = false, found_6 = false;
332     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
333         found6_k4 = false;
334 
335     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
336     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
337       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
338         found_1 = found_1 ||
339             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
340         found_2 = found_2 ||
341             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
342         found_3 = found_3 ||
343             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
344                 != null);
345         found_5 = found_5 ||
346             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
347 
348         CoprocessorEnvironment env =
349             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
350         if (env != null) {
351           found_6 = true;
352           Configuration conf = env.getConfiguration();
353           found6_k1 = conf.get("k1") != null;
354           found6_k2 = conf.get("k2") != null;
355           found6_k3 = conf.get("k3") != null;
356         }
357       }
358     }
359 
360     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
361     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
362     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
363     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
364     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
365 
366     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
367     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
368     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
369     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
370   }
371 
372   @Test
373   public void testClassLoadingFromLibDirInJar() throws Exception {
374     loadingClassFromLibDirInJar("/lib/");
375   }
376 
377   @Test
378   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
379     loadingClassFromLibDirInJar("lib/");
380   }
381 
382   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
383     FileSystem fs = cluster.getFileSystem();
384 
385     File innerJarFile1 = buildCoprocessorJar(cpName1);
386     File innerJarFile2 = buildCoprocessorJar(cpName2);
387     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
388 
389     ClassLoaderTestHelper.addJarFilesToJar(
390       outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
391 
392     // copy the jars into dfs
393     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
394       new Path(fs.getUri().toString() + Path.SEPARATOR));
395     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
396       outerJarFile.getName();
397     assertTrue("Copy jar file to HDFS failed.",
398       fs.exists(new Path(jarFileOnHDFS)));
399     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
400 
401     // create a table that references the coprocessors
402     HTableDescriptor htd = new HTableDescriptor(tableName);
403     htd.addFamily(new HColumnDescriptor("test"));
404       // without configuration values
405     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
406       "|" + Coprocessor.PRIORITY_USER);
407       // with configuration values
408     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
409       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
410     Admin admin = TEST_UTIL.getHBaseAdmin();
411     if (admin.tableExists(tableName)) {
412       if (admin.isTableEnabled(tableName)) {
413         admin.disableTable(tableName);
414       }
415       admin.deleteTable(tableName);
416     }
417     admin.createTable(htd);
418     waitForTable(htd.getTableName());
419 
420     // verify that the coprocessors were loaded
421     boolean found1 = false, found2 = false, found2_k1 = false,
422         found2_k2 = false, found2_k3 = false;
423     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
424     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
425       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
426         CoprocessorEnvironment env;
427         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
428         if (env != null) {
429           found1 = true;
430         }
431         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
432         if (env != null) {
433           found2 = true;
434           Configuration conf = env.getConfiguration();
435           found2_k1 = conf.get("k1") != null;
436           found2_k2 = conf.get("k2") != null;
437           found2_k3 = conf.get("k3") != null;
438         }
439       }
440     }
441     assertTrue("Class " + cpName1 + " was missing on a region", found1);
442     assertTrue("Class " + cpName2 + " was missing on a region", found2);
443     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
444     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
445     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
446   }
447 
448   @Test
449   public void testRegionServerCoprocessorsReported() throws Exception {
450     // This was a test for HBASE-4070.
451     // We are removing coprocessors from region load in HBASE-5258.
452     // Therefore, this test now only checks system coprocessors.
453     assertAllRegionServers(null);
454   }
455 
456   /**
457    * return the subset of all regionservers
458    * (actually returns set of ServerLoads)
459    * which host some region in a given table.
460    * used by assertAllRegionServers() below to
461    * test reporting of loaded coprocessors.
462    * @param tableName : given table.
463    * @return subset of all servers.
464    */
465   Map<ServerName, ServerLoad> serversForTable(String tableName) {
466     Map<ServerName, ServerLoad> serverLoadHashMap =
467         new HashMap<ServerName, ServerLoad>();
468     for(Map.Entry<ServerName,ServerLoad> server:
469         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
470             getOnlineServers().entrySet()) {
471       for( Map.Entry<byte[], RegionLoad> region:
472           server.getValue().getRegionsLoad().entrySet()) {
473         if (region.getValue().getNameAsString().equals(tableName)) {
474           // this server hosts a region of tableName: add this server..
475           serverLoadHashMap.put(server.getKey(),server.getValue());
476           // .. and skip the rest of the regions that it hosts.
477           break;
478         }
479       }
480     }
481     return serverLoadHashMap;
482   }
483 
484   void assertAllRegionServers(String tableName) throws InterruptedException {
485     Map<ServerName, ServerLoad> servers;
486     String[] actualCoprocessors = null;
487     boolean success = false;
488     String[] expectedCoprocessors = regionServerSystemCoprocessors;
489     if (tableName == null) {
490       // if no tableName specified, use all servers.
491       servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
492     } else {
493       servers = serversForTable(tableName);
494     }
495     for (int i = 0; i < 5; i++) {
496       boolean any_failed = false;
497       for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
498         actualCoprocessors = server.getValue().getRsCoprocessors();
499         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
500           LOG.debug("failed comparison: actual: " +
501               Arrays.toString(actualCoprocessors) +
502               " ; expected: " + Arrays.toString(expectedCoprocessors));
503           any_failed = true;
504           expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
505           break;
506         }
507         expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
508       }
509       if (any_failed == false) {
510         success = true;
511         break;
512       }
513       LOG.debug("retrying after failed comparison: " + i);
514       Thread.sleep(1000);
515     }
516     assertTrue(success);
517   }
518 
519   private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
520     if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
521       expectedCoprocessors = masterRegionServerSystemCoprocessors;
522     } else {
523       expectedCoprocessors = regionServerSystemCoprocessors;
524     }
525     return expectedCoprocessors;
526   }
527 
528   @Test
529   public void testMasterCoprocessorsReported() {
530     // HBASE 4070: Improve region server metrics to report loaded coprocessors
531     // to master: verify that the master is reporting the correct set of
532     // loaded coprocessors.
533     final String loadedMasterCoprocessorsVerify =
534         "[" + masterCoprocessor.getSimpleName() + "]";
535     String loadedMasterCoprocessors =
536         java.util.Arrays.toString(
537             TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessors());
538     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
539   }
540 
541   @Test
542   public void testFindCoprocessors() {
543     // HBASE 12277: 
544     CoprocessorHost masterCpHost =
545                              TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost();
546 
547     List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
548 
549     assertTrue(masterObservers != null && masterObservers.size() > 0);
550     assertEquals(masterCoprocessor.getSimpleName(),
551                  masterObservers.get(0).getClass().getSimpleName());
552   }
553 
554   private void waitForTable(TableName name) throws InterruptedException, IOException {
555     // First wait until all regions are online
556     TEST_UTIL.waitTableEnabled(name);
557     // Now wait a bit longer for the coprocessor hosts to load the CPs
558     Thread.sleep(1000);
559   }
560 
561 }
562