parameters: (self, path, username=whoami, keyspace='salvus', passwd=True, password=None)
docstring: - passwd -- if False, don't ask for a password; in this case nothing must require sudo to
            run, and all logins must work using ssh with keys
docstring_summary: - passwd -- if False, don't ask for a password; in this case nothing must require sudo to
            run, and all logins must work using ssh with keys
docstring_tokens: - passwd -- if False don t ask for a password ; in this case nothing must require sudo to run and all logins must work using ssh with keys
function: def __init__(self, path, username=whoami, keyspace='salvus', passwd=True, password=None):
          """
          - passwd -- if False, don't ask for a password; in this case nothing must require sudo to
            run, and all logins must work using ssh with keys
          """
          self._keyspace = keyspace
          self._path = path
          self._username = username
          self._hosts = Hosts(os.path.join(path, 'hosts'), username=username, passwd=passwd, password=password)
  
          self._services, self._ordered_service_names = parse_groupfile(os.path.join(path, 'services'))
          del self._services[None]
  
          self.monitor = Monitor(Hosts(os.path.join(path, 'hosts'), username=username, passwd=False), services = self)
  
          # this is the canonical list of options, expanded out by service and host.
          def hostopts(service, query='all', copy=True):
              """Return list of pairs (hostname, options) defined in the services file, where
              the hostname matches the given hostname/group"""
              restrict = set(self._hosts[query])
              return sum([[(h, dict(opts) if copy else opts) for h in self._hosts[query] if h in restrict]
                                 for query, opts in self._services[service]], [])
  
          self._options = dict([(service, hostopts(service)) for service in self._ordered_service_names])
  
  
          # Divide up hosts by their data center
          self._ip_address_to_dc = {}
          for k, v in self._hosts._ip_addresses.iteritems():
              if len(k) == 3 and k.startswith('dc') and k[2] >= '0' and k[2] <= '9':
                  dc = int(k[2])
                  for w in v:
                      self._ip_address_to_dc[w] = dc
  
  
          ##########################################
          # Programatically fill in extra options to the list
          ##########################################
          # CASSANDRA options
          self._cassandras_in_dc = {}
          if 'cassandra' in self._options:
              v = self._options['cassandra']
              # determine the seeds
              seeds = ','.join([self._hosts.hostname(h) for h, o in v if o.get('seed',False)])
              # determine global topology file; ip_address=data_center:rack
              topology = '\n'.join(['%s=%s'%(self._hosts.hostname(h), o.get('topology', 'DC0:RAC0'))
                                                                    for h, o in v] + ['default=DC0:RAC0'])
  
              for address, o in v:
                  dc = o.get('topology','DC0:RAC0').split(':')[0].lower()  # this must be before o['topology'] line below!
                  if dc not in self._cassandras_in_dc:
                      self._cassandras_in_dc[dc] = [self._hosts.hostname(address)]
                  else:
                      self._cassandras_in_dc[dc].append(self._hosts.hostname(address))
                  o['seeds']          = seeds
                  o['topology']       = topology
                  o['listen_address'] = self._hosts.hostname(address)
                  o['rpc_address']    = self._hosts.hostname(address)
                  if 'seed' in o: del o['seed']
  
              native_transport_port = v[0][1].get('native_transport_port', 9042)
              if native_transport_port != 9042:
                  print "Serving cassandra on non-standard port %s"%native_transport_port
              try:
                  self._cassandra = ['%s:%s'%(self._hosts.hostname(h), native_transport_port) for h in self._hosts['cassandra']]
                  import cassandra
                  cassandra.KEYSPACE = self._keyspace
                  cassandra.set_nodes(self._hosts['cassandra'])
              except ValueError:
                  print "WARNING: no cassandra hosts -- severly degraded functionality!"
  
          # HAPROXY options
          if 'haproxy' in self._options:
              nginx_servers = [{'ip':h,'port':o.get('port',NGINX_PORT), 'maxconn':10000}
                               for h, o in self._options['nginx']]
              hub_servers = [{'ip':h,'port':o.get('port',HUB_PORT), 'maxconn':100}
                                for h, o in self._options['hub']]
              # NOTE: right now we assume that the proxy servers are running on exactly the same machine as the hub,
              # since they are implemented as part of the same process (though, listening on a different ports).
              proxy_servers = [{'ip':h,'proxy_port':o.get('proxy_port',HUB_PROXY_PORT), 'maxconn':100}
                                for h, o in self._options['hub']]
              for _, o in self._options['haproxy']:
                  if 'nginx_servers' not in o:
                      o['nginx_servers'] = nginx_servers
                  if 'hub_servers' not in o:
                      o['hub_servers'] = hub_servers
                  if 'proxy_servers' not in o:
                      o['proxy_servers'] = proxy_servers
  
          # HUB options
          if 'hub' in self._options:
              for host, o in self._options['hub']:
                  # very important: set to listen only on our VPN.
                  #o['host'] = host
                  # FOR GCE: since everything is on a vpn anyways fastest to listen on everything
                  o['host'] = self._hosts.hostname(host)
  
          # Syncstring options
          #if 'syncstring' in self._options:
          #    for host, o in self._options['syncstring']:
          #        # set to listen only on our VPN -- slight extra security
          #        o['host'] = host
  
          # COMPUTE options
          if 'compute' in self._options:
              for host, o in self._options['compute']:
                  # Very, very important: set to listen only on our VPN!
                  # There is an attack where a local user
                  # can bind to a more specific host and same port on a
                  # machine, and intercept all trafic.  This would mean
                  # they could effectively man-in-the-middle take over a
                  # node.  By binding on a specific ip address, we
                  # prevent this possibility.  I haven't tried this, so
                  # I'm not 100% sure.
                  o['host'] = host
  
          if 'sagenb' in self._options:
              for address, o in self._options['sagenb']:
                  o['address'] = address
  
          # VM options
          for t in ['vm','vmgce']:
              if t in self._options:
                  for address, o in self._options[t]:
                      # very, very important: set to listen only on our VPN!  There is an attack where a local user
                      # can bind to a more specific address and same port on a machine, and intercept all trafic.
                      if 'ip_address' not in o:
                          addresses = self._hosts[o['hostname']]
                          if len(addresses) != 1:
                              raise RuntimeError("Error configuring a VM: hostname %s doesn't uniquely determine one ip address"%o['hostname'])
                          o['ip_address'] = addresses[0]
function_tokens: def __init__ ( self , path , username = whoami , keyspace = 'salvus' , passwd = True , password = None ) : self . _keyspace = keyspace self . _path = path self . _username = username self . _hosts = Hosts ( os . path . join ( path , 'hosts' ) , username = username , passwd = passwd , password = password ) self . _services , self . _ordered_service_names = parse_groupfile ( os . path . join ( path , 'services' ) ) del self . _services [ None ] self . monitor = Monitor ( Hosts ( os . path . join ( path , 'hosts' ) , username = username , passwd = False ) , services = self ) # this is the canonical list of options, expanded out by service and host. def hostopts ( service , query = 'all' , copy = True ) : """Return list of pairs (hostname, options) defined in the services file, where
              the hostname matches the given hostname/group""" restrict = set ( self . _hosts [ query ] ) return sum ( [ [ ( h , dict ( opts ) if copy else opts ) for h in self . _hosts [ query ] if h in restrict ] for query , opts in self . _services [ service ] ] , [ ] ) self . _options = dict ( [ ( service , hostopts ( service ) ) for service in self . _ordered_service_names ] ) # Divide up hosts by their data center self . _ip_address_to_dc = { } for k , v in self . _hosts . _ip_addresses . iteritems ( ) : if len ( k ) == 3 and k . startswith ( 'dc' ) and k [ 2 ] >= '0' and k [ 2 ] <= '9' : dc = int ( k [ 2 ] ) for w in v : self . _ip_address_to_dc [ w ] = dc ########################################## # Programatically fill in extra options to the list ########################################## # CASSANDRA options self . _cassandras_in_dc = { } if 'cassandra' in self . _options : v = self . _options [ 'cassandra' ] # determine the seeds seeds = ',' . join ( [ self . _hosts . hostname ( h ) for h , o in v if o . get ( 'seed' , False ) ] ) # determine global topology file; ip_address=data_center:rack topology = '\n' . join ( [ '%s=%s' % ( self . _hosts . hostname ( h ) , o . get ( 'topology' , 'DC0:RAC0' ) ) for h , o in v ] + [ 'default=DC0:RAC0' ] ) for address , o in v : dc = o . get ( 'topology' , 'DC0:RAC0' ) . split ( ':' ) [ 0 ] . lower ( ) # this must be before o['topology'] line below! if dc not in self . _cassandras_in_dc : self . _cassandras_in_dc [ dc ] = [ self . _hosts . hostname ( address ) ] else : self . _cassandras_in_dc [ dc ] . append ( self . _hosts . hostname ( address ) ) o [ 'seeds' ] = seeds o [ 'topology' ] = topology o [ 'listen_address' ] = self . _hosts . hostname ( address ) o [ 'rpc_address' ] = self . _hosts . hostname ( address ) if 'seed' in o : del o [ 'seed' ] native_transport_port = v [ 0 ] [ 1 ] . get ( 'native_transport_port' , 9042 ) if native_transport_port != 9042 : print "Serving cassandra on non-standard port %s" % native_transport_port try : self . _cassandra = [ '%s:%s' % ( self . _hosts . hostname ( h ) , native_transport_port ) for h in self . _hosts [ 'cassandra' ] ] import cassandra cassandra . KEYSPACE = self . _keyspace cassandra . set_nodes ( self . _hosts [ 'cassandra' ] ) except ValueError : print "WARNING: no cassandra hosts -- severly degraded functionality!" # HAPROXY options if 'haproxy' in self . _options : nginx_servers = [ { 'ip' : h , 'port' : o . get ( 'port' , NGINX_PORT ) , 'maxconn' : 10000 } for h , o in self . _options [ 'nginx' ] ] hub_servers = [ { 'ip' : h , 'port' : o . get ( 'port' , HUB_PORT ) , 'maxconn' : 100 } for h , o in self . _options [ 'hub' ] ] # NOTE: right now we assume that the proxy servers are running on exactly the same machine as the hub, # since they are implemented as part of the same process (though, listening on a different ports). proxy_servers = [ { 'ip' : h , 'proxy_port' : o . get ( 'proxy_port' , HUB_PROXY_PORT ) , 'maxconn' : 100 } for h , o in self . _options [ 'hub' ] ] for _ , o in self . _options [ 'haproxy' ] : if 'nginx_servers' not in o : o [ 'nginx_servers' ] = nginx_servers if 'hub_servers' not in o : o [ 'hub_servers' ] = hub_servers if 'proxy_servers' not in o : o [ 'proxy_servers' ] = proxy_servers # HUB options if 'hub' in self . _options : for host , o in self . _options [ 'hub' ] : # very important: set to listen only on our VPN. #o['host'] = host # FOR GCE: since everything is on a vpn anyways fastest to listen on everything o [ 'host' ] = self . _hosts . hostname ( host ) # Syncstring options #if 'syncstring' in self._options: #    for host, o in self._options['syncstring']: #        # set to listen only on our VPN -- slight extra security #        o['host'] = host # COMPUTE options if 'compute' in self . _options : for host , o in self . _options [ 'compute' ] : # Very, very important: set to listen only on our VPN! # There is an attack where a local user # can bind to a more specific host and same port on a # machine, and intercept all trafic.  This would mean # they could effectively man-in-the-middle take over a # node.  By binding on a specific ip address, we # prevent this possibility.  I haven't tried this, so # I'm not 100% sure. o [ 'host' ] = host if 'sagenb' in self . _options : for address , o in self . _options [ 'sagenb' ] : o [ 'address' ] = address # VM options for t in [ 'vm' , 'vmgce' ] : if t in self . _options : for address , o in self . _options [ t ] : # very, very important: set to listen only on our VPN!  There is an attack where a local user # can bind to a more specific address and same port on a machine, and intercept all trafic. if 'ip_address' not in o : addresses = self . _hosts [ o [ 'hostname' ] ] if len ( addresses ) != 1 : raise RuntimeError ( "Error configuring a VM: hostname %s doesn't uniquely determine one ip address" % o [ 'hostname' ] ) o [ 'ip_address' ] = addresses [ 0 ]