View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.balancer;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.mockito.Mockito.mock;
23  import static org.mockito.Mockito.when;
24  
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.LinkedHashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeMap;
32  import java.util.TreeSet;
33  
34  import org.apache.commons.lang.ArrayUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.testclassification.MediumTests;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
44  import org.apache.hadoop.hbase.master.LoadBalancer;
45  import org.apache.hadoop.hbase.master.MasterServices;
46  import org.apache.hadoop.hbase.master.RackManager;
47  import org.apache.hadoop.hbase.master.RegionPlan;
48  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
49  import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction;
50  import org.apache.hadoop.net.DNSToSwitchMapping;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  import org.mockito.Mockito;
55  
56  import com.google.common.collect.Lists;
57  
58  @Category(MediumTests.class)
59  public class TestBaseLoadBalancer extends BalancerTestBase {
60  
61    private static LoadBalancer loadBalancer;
62    private static final Log LOG = LogFactory.getLog(TestBaseLoadBalancer.class);
63    private static final ServerName master = ServerName.valueOf("fake-master", 0, 1L);
64    private static RackManager rackManager;
65    private static final int NUM_SERVERS = 15;
66    private static ServerName[] servers = new ServerName[NUM_SERVERS];
67  
68    int[][] regionsAndServersMocks = new int[][] {
69        // { num regions, num servers }
70        new int[] { 0, 0 }, new int[] { 0, 1 }, new int[] { 1, 1 }, new int[] { 2, 1 },
71        new int[] { 10, 1 }, new int[] { 1, 2 }, new int[] { 2, 2 }, new int[] { 3, 2 },
72        new int[] { 1, 3 }, new int[] { 2, 3 }, new int[] { 3, 3 }, new int[] { 25, 3 },
73        new int[] { 2, 10 }, new int[] { 2, 100 }, new int[] { 12, 10 }, new int[] { 12, 100 }, };
74  
75    @BeforeClass
76    public static void beforeAllTests() throws Exception {
77      Configuration conf = HBaseConfiguration.create();
78      conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
79      loadBalancer = new MockBalancer();
80      loadBalancer.setConf(conf);
81      MasterServices st = Mockito.mock(MasterServices.class);
82      Mockito.when(st.getServerName()).thenReturn(master);
83      loadBalancer.setMasterServices(st);
84  
85      // Set up the rack topologies (5 machines per rack)
86      rackManager = Mockito.mock(RackManager.class);
87      for (int i = 0; i < NUM_SERVERS; i++) {
88        servers[i] = ServerName.valueOf("foo"+i+":1234",-1);
89        if (i < 5) {
90          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack1");
91        }
92        if (i >= 5 && i < 10) {
93          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack2");
94        }
95        if (i >= 10) {
96          Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack3");
97        }
98      }
99    }
100 
101   public static class MockBalancer extends BaseLoadBalancer {
102 
103     @Override
104     public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
105       return null;
106     }
107 
108   }
109 
110   /**
111    * Tests immediate assignment.
112    *
113    * Invariant is that all regions have an assignment.
114    *
115    * @throws Exception
116    */
117   @Test (timeout=30000)
118   public void testImmediateAssignment() throws Exception {
119     for (int[] mock : regionsAndServersMocks) {
120       LOG.debug("testImmediateAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
121       List<HRegionInfo> regions = randomRegions(mock[0]);
122       List<ServerAndLoad> servers = randomServers(mock[1], 0);
123       List<ServerName> list = getListOfServerNames(servers);
124       Map<HRegionInfo, ServerName> assignments = loadBalancer.immediateAssignment(regions, list);
125       assertImmediateAssignment(regions, list, assignments);
126       returnRegions(regions);
127       returnServers(list);
128     }
129   }
130 
131   /**
132    * All regions have an assignment.
133    * @param regions
134    * @param servers
135    * @param assignments
136    */
137   private void assertImmediateAssignment(List<HRegionInfo> regions, List<ServerName> servers,
138       Map<HRegionInfo, ServerName> assignments) {
139     for (HRegionInfo region : regions) {
140       assertTrue(assignments.containsKey(region));
141     }
142   }
143 
144   /**
145    * Tests the bulk assignment used during cluster startup.
146    *
147    * Round-robin. Should yield a balanced cluster so same invariant as the load
148    * balancer holds, all servers holding either floor(avg) or ceiling(avg).
149    *
150    * @throws Exception
151    */
152   @Test (timeout=180000)
153   public void testBulkAssignment() throws Exception {
154     for (int[] mock : regionsAndServersMocks) {
155       LOG.debug("testBulkAssignment with " + mock[0] + " regions and " + mock[1] + " servers");
156       List<HRegionInfo> regions = randomRegions(mock[0]);
157       List<ServerAndLoad> servers = randomServers(mock[1], 0);
158       List<ServerName> list = getListOfServerNames(servers);
159       Map<ServerName, List<HRegionInfo>> assignments =
160           loadBalancer.roundRobinAssignment(regions, list);
161       float average = (float) regions.size() / servers.size();
162       int min = (int) Math.floor(average);
163       int max = (int) Math.ceil(average);
164       if (assignments != null && !assignments.isEmpty()) {
165         for (List<HRegionInfo> regionList : assignments.values()) {
166           assertTrue(regionList.size() == min || regionList.size() == max);
167         }
168       }
169       returnRegions(regions);
170       returnServers(list);
171     }
172   }
173 
174   /**
175    * Test the cluster startup bulk assignment which attempts to retain
176    * assignment info.
177    * @throws Exception
178    */
179   @Test (timeout=180000)
180   public void testRetainAssignment() throws Exception {
181     // Test simple case where all same servers are there
182     List<ServerAndLoad> servers = randomServers(10, 10);
183     List<HRegionInfo> regions = randomRegions(100);
184     Map<HRegionInfo, ServerName> existing = new TreeMap<HRegionInfo, ServerName>();
185     for (int i = 0; i < regions.size(); i++) {
186       ServerName sn = servers.get(i % servers.size()).getServerName();
187       // The old server would have had same host and port, but different
188       // start code!
189       ServerName snWithOldStartCode =
190           ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10);
191       existing.put(regions.get(i), snWithOldStartCode);
192     }
193     List<ServerName> listOfServerNames = getListOfServerNames(servers);
194     Map<ServerName, List<HRegionInfo>> assignment =
195         loadBalancer.retainAssignment(existing, listOfServerNames);
196     assertRetainedAssignment(existing, listOfServerNames, assignment);
197 
198     // Include two new servers that were not there before
199     List<ServerAndLoad> servers2 = new ArrayList<ServerAndLoad>(servers);
200     servers2.add(randomServer(10));
201     servers2.add(randomServer(10));
202     listOfServerNames = getListOfServerNames(servers2);
203     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
204     assertRetainedAssignment(existing, listOfServerNames, assignment);
205 
206     // Remove two of the servers that were previously there
207     List<ServerAndLoad> servers3 = new ArrayList<ServerAndLoad>(servers);
208     servers3.remove(0);
209     servers3.remove(0);
210     listOfServerNames = getListOfServerNames(servers3);
211     assignment = loadBalancer.retainAssignment(existing, listOfServerNames);
212     assertRetainedAssignment(existing, listOfServerNames, assignment);
213   }
214 
215   @Test (timeout=180000)
216   public void testRegionAvailability() throws Exception {
217     // Create a cluster with a few servers, assign them to specific racks
218     // then assign some regions. The tests should check whether moving a
219     // replica from one node to a specific other node or rack lowers the
220     // availability of the region or not
221 
222     List<HRegionInfo> list0 = new ArrayList<HRegionInfo>();
223     List<HRegionInfo> list1 = new ArrayList<HRegionInfo>();
224     List<HRegionInfo> list2 = new ArrayList<HRegionInfo>();
225     // create a region (region1)
226     HRegionInfo hri1 = new HRegionInfo(
227         TableName.valueOf("table"), "key1".getBytes(), "key2".getBytes(),
228         false, 100);
229     // create a replica of the region (replica_of_region1)
230     HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
231     // create a second region (region2)
232     HRegionInfo hri3 = new HRegionInfo(
233         TableName.valueOf("table"), "key2".getBytes(), "key3".getBytes(),
234         false, 101);
235     list0.add(hri1); //only region1
236     list1.add(hri2); //only replica_of_region1
237     list2.add(hri3); //only region2
238     Map<ServerName, List<HRegionInfo>> clusterState =
239         new LinkedHashMap<ServerName, List<HRegionInfo>>();
240     clusterState.put(servers[0], list0); //servers[0] hosts region1
241     clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
242     clusterState.put(servers[2], list2); //servers[2] hosts region2
243     // create a cluster with the above clusterState. The way in which the
244     // cluster is created (constructor code) would make sure the indices of
245     // the servers are in the order in which it is inserted in the clusterState
246     // map (linkedhashmap is important). A similar thing applies to the region lists
247     Cluster cluster = new Cluster(clusterState, null, null, rackManager);
248     // check whether a move of region1 from servers[0] to servers[1] would lower
249     // the availability of region1
250     assertTrue(cluster.wouldLowerAvailability(hri1, servers[1]));
251     // check whether a move of region1 from servers[0] to servers[2] would lower
252     // the availability of region1
253     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
254     // check whether a move of replica_of_region1 from servers[0] to servers[2] would lower
255     // the availability of replica_of_region1
256     assertTrue(!cluster.wouldLowerAvailability(hri2, servers[2]));
257     // check whether a move of region2 from servers[0] to servers[1] would lower
258     // the availability of region2
259     assertTrue(!cluster.wouldLowerAvailability(hri3, servers[1]));
260 
261     // now lets have servers[1] host replica_of_region2
262     list1.add(RegionReplicaUtil.getRegionInfoForReplica(hri3, 1));
263     // create a new clusterState with the above change
264     cluster = new Cluster(clusterState, null, null, rackManager);
265     // now check whether a move of a replica from servers[0] to servers[1] would lower
266     // the availability of region2
267     assertTrue(cluster.wouldLowerAvailability(hri3, servers[1]));
268 
269     // start over again
270     clusterState.clear();
271     clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
272     clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1 and replica_of_region2
273     clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
274     clusterState.put(servers[10], new ArrayList<HRegionInfo>()); //servers[10], rack3 hosts no region
275     // create a cluster with the above clusterState
276     cluster = new Cluster(clusterState, null, null, rackManager);
277     // check whether a move of region1 from servers[0],rack1 to servers[6],rack2 would
278     // lower the availability
279 
280     assertTrue(cluster.wouldLowerAvailability(hri1, servers[0]));
281 
282     // now create a cluster without the rack manager
283     cluster = new Cluster(clusterState, null, null, null);
284     // now repeat check whether a move of region1 from servers[0] to servers[6] would
285     // lower the availability
286     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[6]));
287   }
288 
289   @Test (timeout=180000)
290   public void testRegionAvailabilityWithRegionMoves() throws Exception {
291     List<HRegionInfo> list0 = new ArrayList<HRegionInfo>();
292     List<HRegionInfo> list1 = new ArrayList<HRegionInfo>();
293     List<HRegionInfo> list2 = new ArrayList<HRegionInfo>();
294     // create a region (region1)
295     HRegionInfo hri1 = new HRegionInfo(
296         TableName.valueOf("table"), "key1".getBytes(), "key2".getBytes(),
297         false, 100);
298     // create a replica of the region (replica_of_region1)
299     HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
300     // create a second region (region2)
301     HRegionInfo hri3 = new HRegionInfo(
302         TableName.valueOf("table"), "key2".getBytes(), "key3".getBytes(),
303         false, 101);
304     list0.add(hri1); //only region1
305     list1.add(hri2); //only replica_of_region1
306     list2.add(hri3); //only region2
307     Map<ServerName, List<HRegionInfo>> clusterState =
308         new LinkedHashMap<ServerName, List<HRegionInfo>>();
309     clusterState.put(servers[0], list0); //servers[0] hosts region1
310     clusterState.put(servers[1], list1); //servers[1] hosts replica_of_region1
311     clusterState.put(servers[2], list2); //servers[2] hosts region2
312     // create a cluster with the above clusterState. The way in which the
313     // cluster is created (constructor code) would make sure the indices of
314     // the servers are in the order in which it is inserted in the clusterState
315     // map (linkedhashmap is important).
316     Cluster cluster = new Cluster(clusterState, null, null, rackManager);
317     // check whether moving region1 from servers[1] to servers[2] would lower availability
318     assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
319 
320     // now move region1 from servers[0] to servers[2]
321     cluster.doAction(new MoveRegionAction(0, 0, 2));
322     // now repeat check whether moving region1 from servers[1] to servers[2]
323     // would lower availability
324     assertTrue(cluster.wouldLowerAvailability(hri1, servers[2]));
325 
326     // start over again
327     clusterState.clear();
328     List<HRegionInfo> list3 = new ArrayList<HRegionInfo>();
329     HRegionInfo hri4 = RegionReplicaUtil.getRegionInfoForReplica(hri3, 1);
330     list3.add(hri4);
331     clusterState.put(servers[0], list0); //servers[0], rack1 hosts region1
332     clusterState.put(servers[5], list1); //servers[5], rack2 hosts replica_of_region1
333     clusterState.put(servers[6], list2); //servers[6], rack2 hosts region2
334     clusterState.put(servers[12], list3); //servers[12], rack3 hosts replica_of_region2
335     // create a cluster with the above clusterState
336     cluster = new Cluster(clusterState, null, null, rackManager);
337     // check whether a move of replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
338     // lower the availability
339     assertTrue(!cluster.wouldLowerAvailability(hri4, servers[0]));
340     // now move region2 from servers[6],rack2 to servers[0],rack1
341     cluster.doAction(new MoveRegionAction(2, 2, 0));
342     // now repeat check if replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
343     // lower the availability
344     assertTrue(cluster.wouldLowerAvailability(hri3, servers[0]));
345   }
346 
347   private List<ServerName> getListOfServerNames(final List<ServerAndLoad> sals) {
348     List<ServerName> list = new ArrayList<ServerName>();
349     for (ServerAndLoad e : sals) {
350       list.add(e.getServerName());
351     }
352     return list;
353   }
354 
355   /**
356    * Asserts a valid retained assignment plan.
357    * <p>
358    * Must meet the following conditions:
359    * <ul>
360    * <li>Every input region has an assignment, and to an online server
361    * <li>If a region had an existing assignment to a server with the same
362    * address a a currently online server, it will be assigned to it
363    * </ul>
364    * @param existing
365    * @param servers
366    * @param assignment
367    */
368   private void assertRetainedAssignment(Map<HRegionInfo, ServerName> existing,
369       List<ServerName> servers, Map<ServerName, List<HRegionInfo>> assignment) {
370     // Verify condition 1, every region assigned, and to online server
371     Set<ServerName> onlineServerSet = new TreeSet<ServerName>(servers);
372     Set<HRegionInfo> assignedRegions = new TreeSet<HRegionInfo>();
373     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
374       assertTrue("Region assigned to server that was not listed as online",
375         onlineServerSet.contains(a.getKey()));
376       for (HRegionInfo r : a.getValue())
377         assignedRegions.add(r);
378     }
379     assertEquals(existing.size(), assignedRegions.size());
380 
381     // Verify condition 2, if server had existing assignment, must have same
382     Set<String> onlineHostNames = new TreeSet<String>();
383     for (ServerName s : servers) {
384       onlineHostNames.add(s.getHostname());
385     }
386 
387     for (Map.Entry<ServerName, List<HRegionInfo>> a : assignment.entrySet()) {
388       ServerName assignedTo = a.getKey();
389       for (HRegionInfo r : a.getValue()) {
390         ServerName address = existing.get(r);
391         if (address != null && onlineHostNames.contains(address.getHostname())) {
392           // this region was prevously assigned somewhere, and that
393           // host is still around, then it should be re-assigned on the
394           // same host
395           assertEquals(address.getHostname(), assignedTo.getHostname());
396         }
397       }
398     }
399   }
400 
401   @Test (timeout=180000)
402   public void testClusterServersWithSameHostPort() {
403     // tests whether the BaseLoadBalancer.Cluster can be constructed with servers
404     // sharing same host and port
405     List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
406     List<HRegionInfo> regions = randomRegions(101);
407     Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName, List<HRegionInfo>>();
408 
409     assignRegions(regions, servers, clusterState);
410 
411     // construct another list of servers, but sharing same hosts and ports
412     List<ServerName> oldServers = new ArrayList<ServerName>(servers.size());
413     for (ServerName sn : servers) {
414       // The old server would have had same host and port, but different start code!
415       oldServers.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 10));
416     }
417 
418     regions = randomRegions(9); // some more regions
419     assignRegions(regions, oldServers, clusterState);
420 
421     // should not throw exception:
422     BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, null, null);
423     assertEquals(101 + 9, cluster.numRegions);
424     assertEquals(10, cluster.numServers); // only 10 servers because they share the same host + port
425   }
426 
427   private void assignRegions(List<HRegionInfo> regions, List<ServerName> servers,
428       Map<ServerName, List<HRegionInfo>> clusterState) {
429     for (int i = 0; i < regions.size(); i++) {
430       ServerName sn = servers.get(i % servers.size());
431       List<HRegionInfo> regionsOfServer = clusterState.get(sn);
432       if (regionsOfServer == null) {
433         regionsOfServer = new ArrayList<HRegionInfo>(10);
434         clusterState.put(sn, regionsOfServer);
435       }
436 
437       regionsOfServer.add(regions.get(i));
438     }
439   }
440 
441   @Test (timeout=180000)
442   public void testClusterRegionLocations() {
443     // tests whether region locations are handled correctly in Cluster
444     List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
445     List<HRegionInfo> regions = randomRegions(101);
446     Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName, List<HRegionInfo>>();
447 
448     assignRegions(regions, servers, clusterState);
449 
450     // mock block locality for some regions
451     RegionLocationFinder locationFinder = mock(RegionLocationFinder.class);
452     // block locality: region:0   => {server:0}
453     //                 region:1   => {server:0, server:1}
454     //                 region:42 => {server:4, server:9, server:5}
455     when(locationFinder.getTopBlockLocations(regions.get(0))).thenReturn(
456       Lists.newArrayList(servers.get(0)));
457     when(locationFinder.getTopBlockLocations(regions.get(1))).thenReturn(
458       Lists.newArrayList(servers.get(0), servers.get(1)));
459     when(locationFinder.getTopBlockLocations(regions.get(42))).thenReturn(
460       Lists.newArrayList(servers.get(4), servers.get(9), servers.get(5)));
461     when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn(
462       Lists.newArrayList(ServerName.valueOf("foo", 0, 0))); // this server does not exists in clusterStatus
463 
464     BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, locationFinder, null);
465 
466     int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just a test
467     int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));
468     int r10 = ArrayUtils.indexOf(cluster.regions, regions.get(10));
469     int r42 = ArrayUtils.indexOf(cluster.regions, regions.get(42));
470     int r43 = ArrayUtils.indexOf(cluster.regions, regions.get(43));
471 
472     int s0 = cluster.serversToIndex.get(servers.get(0).getHostAndPort());
473     int s1 = cluster.serversToIndex.get(servers.get(1).getHostAndPort());
474     int s4 = cluster.serversToIndex.get(servers.get(4).getHostAndPort());
475     int s5 = cluster.serversToIndex.get(servers.get(5).getHostAndPort());
476     int s9 = cluster.serversToIndex.get(servers.get(9).getHostAndPort());
477 
478     // region 0 locations
479     assertEquals(1, cluster.regionLocations[r0].length);
480     assertEquals(s0, cluster.regionLocations[r0][0]);
481 
482     // region 1 locations
483     assertEquals(2, cluster.regionLocations[r1].length);
484     assertEquals(s0, cluster.regionLocations[r1][0]);
485     assertEquals(s1, cluster.regionLocations[r1][1]);
486 
487     // region 10 locations
488     assertEquals(0, cluster.regionLocations[r10].length);
489 
490     // region 42 locations
491     assertEquals(3, cluster.regionLocations[r42].length);
492     assertEquals(s4, cluster.regionLocations[r42][0]);
493     assertEquals(s9, cluster.regionLocations[r42][1]);
494     assertEquals(s5, cluster.regionLocations[r42][2]);
495 
496     // region 43 locations
497     assertEquals(1, cluster.regionLocations[r43].length);
498     assertEquals(-1, cluster.regionLocations[r43][0]);
499   }
500 }