Python tornado.log.app_log.info() Examples

def manually_kill_server(user_name):
    # Get our AWS server db's instance for the user
    try:
        server = Server.get_server(user_name)
        app_log.debug("Checking server for %s manually..." % user_name)
    except Server.DoesNotExist:
        # it is not necessarily the case that a server will exist, we return early if that is the case.
        app_log.warn("There is no matching, allocated server for user %s" % user_name)
        return
    # get server instance information
    resource = yield retry(boto3.resource, "ec2", region_name=SERVER_PARAMS["REGION"])
    instance = yield retry(resource.Instance, server.server_id)
    # instance object is lazy, run this to get full info...
    yield retry(instance.load)
    
    #stop server if state is running (possible states are stopped, stopping, pending, shutting-down, terminated, and running)
    if instance.state["Name"] == "running":
        retry(instance.stop)
        app_log.info("manually killed server for user %s" % user_name)
    else:
        app_log.debug("server state for user %s is %s, no action taken" % (user_name, instance.state["Name"])) 

def get(self):

		print
		app_log.info("TestBulmaHandler.get... ")

		self.render(
			"test_bulma_extensions.html",
			page_title 				= app_main_texts["main_title"],
			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		)





### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### TORNADO MODULES #########################################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ### 

def get(self):

		app_log.info("InfosCreditsHandler.get... ")

		self.site_section = "infos"

		self.render(
			"credits.html",
			page_title 				= app_main_texts["main_title"],

			app_host				= self.request.host,
			site_section			= self.site_section,
	
			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		)



### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### SNIPPETS handlers #######################################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ### 

def get(self):

		app_log.info("InfosContributeHandler.get... ")

		self.site_section = "infos"

		self.render(
			"contribute.html",
			page_title 				= app_main_texts["main_title"],

			app_host				= self.request.host,
			site_section			= self.site_section,

			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		) 

def get(self):

		app_log.info("InfosStackHandler.get... ")

		self.site_section = "infos"

		self.render(
			"stack.html",
			page_title 				= app_main_texts["main_title"],

			app_host				= self.request.host,
			site_section			= self.site_section,

			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		) 

def get(self):

		app_log.info("InfosWhyHandler.get... ")

		self.site_section = "infos"

		self.render(
			"why.html",
			page_title 				= app_main_texts["main_title"],
			app_host				= self.request.host,
			site_section			= self.site_section,
			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		) 

def post(self, *args):
		"""Example handle ajax post"""
		dic = tornado.escape.json_decode(self.request.body)
		app_log.info("ajax / dic : \n %s " , pformat(dic) )

		# useful code goes here
		
		self.write(json.dumps({'status': 'ok', 'sent': dic}))
		self.finish()




### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### INFOS / DOC  ############################################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ### 

def partial_config_as_dict(self, previous_config=None ) : 
		""" """
		print 
		app_log.info("== SpiderConfig.partial_config_as_dict / previous_config : %s ", previous_config )
		# print "\n*** SpiderConfig.partial_config_as_dict / previous_config : "
		# print previous_config

		all_custom_fields = CONTRIBUTOR_CUSTOMAZIBLE_FIELDS + ["scraper_config_xpaths"]
		partial_config = { k : v for k,v in self.spider_config.iteritems() if k in all_custom_fields }
		# print "\n*** SpiderConfig.partial_config_as_dict / partial_config : "
		# print partial_config
		
		# reset scraper_log
		partial_config["scraper_log"] = deepcopy(CONTRIBUTOR_CORE_FIELDS["scraper_log"])
		partial_config["scraper_log"]["modified_by"]	= self.user
		partial_config["scraper_log"]["modified_at"]	= self.timestamp
		partial_config["scraper_log"]["added_by"] 		= previous_config["scraper_log"]["added_by"]

		return partial_config 

def reset_is_running_on_all_spider( coll_model ) :
	"""
	reset is_running on all spiders to avoid errors if app shut down while one spider was running 
	"""

	print ()

	app_log.warning('>>> reset_is_running_on_all_spider ... ')
	
	# find if any spider was running
	running_spiders = coll_model.find({"scraper_log.is_running" : True})
	app_log.info(">>> running_spiders : \n %s" , list(running_spiders) )

	coll_model.update_many({'scraper_log.is_running' : True }, {"$set": {'scraper_log.is_running' : False }})

	# if list(running_spiders) != [] : 

	# 	app_log.warning('>>> reset_is_running_on_all_spider / some spiders were blocked in is_running == True ... ')
	# 	app_log.warning('>>> spiders are : \n %s', pformat(list(running_spiders)) )

	# 	coll_model.update({"scraper_log.is_running":True}, {"$set" : {"scraper_log.is_running" : False }})
	
	# print 

def _get_pods(self):
        """Get information about build and user pods"""
        app_log.info("Getting pod statistics")
        k8s = self.settings["kubernetes_client"]
        pool = self.settings["executor"]

        get_user_pods = asyncio.wrap_future(
            pool.submit(
                k8s.list_namespaced_pod,
                self.settings["build_namespace"],
                label_selector="app=jupyterhub,component=singleuser-server",
            )
        )

        get_build_pods = asyncio.wrap_future(
            pool.submit(
                k8s.list_namespaced_pod,
                self.settings["build_namespace"],
                label_selector="component=binderhub-build",
            )
        )

        return await asyncio.gather(get_user_pods, get_build_pods) 

def cull_idle(url, api_token, timeout):
    """cull idle single-user servers"""
    auth_header = {
            'Authorization': 'token %s' % api_token
        }
    req = HTTPRequest(url=url + '/users',
        headers=auth_header,
    )
    now = datetime.datetime.utcnow()
    cull_limit = now - datetime.timedelta(seconds=timeout)
    client = AsyncHTTPClient()
    resp = yield client.fetch(req)
    users = json.loads(resp.body.decode('utf8', 'replace'))
    futures = []
    for user in users:
        last_activity = parse_date(user['last_activity'])
        if user['server'] and last_activity < cull_limit:
            app_log.info("Culling %s (inactive since %s)", user['name'], last_activity)
            req = HTTPRequest(url=url + '/users/%s/server' % user['name'],
                method='DELETE',
                headers=auth_header,
            )
            futures.append((user['name'], client.fetch(req)))
        elif user['server'] and last_activity > cull_limit:
            app_log.debug("Not culling %s (active since %s)", user['name'], last_activity)
    
    for (name, f) in futures:
        yield f
        app_log.debug("Finished culling %s", name) 

def stream_logs(self):
        """Stream a pod's logs"""
        app_log.info("Watching logs of %s", self.name)
        for line in self.api.read_namespaced_pod_log(
                self.name,
                self.namespace,
                follow=True,
                tail_lines=self.log_tail_lines,
                _preload_content=False):
            if self.stop_event.is_set():
                app_log.info("Stopping logs of %s", self.name)
                return
            # verify that the line is JSON
            line = line.decode('utf-8')
            try:
                json.loads(line)
            except ValueError:
                # log event wasn't JSON.
                # use the line itself as the message with unknown phase.
                # We don't know what the right phase is, use 'unknown'.
                # If it was a fatal error, presumably a 'failure'
                # message will arrive shortly.
                app_log.error("log event not json: %r", line)
                line = json.dumps({
                    'phase': 'unknown',
                    'message': line,
                })

            self.progress('log', line)
        else:
            app_log.info("Finished streaming logs of %s", self.name) 

def cull_idle(url, api_token, timeout):
    """cull idle single-user servers"""
    auth_header = {
            'Authorization': 'token %s' % api_token
        }
    req = HTTPRequest(url=url + '/users',
        headers=auth_header,
    )
    now = datetime.datetime.utcnow()
    cull_limit = now - datetime.timedelta(seconds=timeout)
    client = AsyncHTTPClient()
    resp = yield client.fetch(req)
    users = json.loads(resp.body.decode('utf8', 'replace'))
    futures = []
    for user in users:
        last_activity = parse_date(user['last_activity'])
        if user['server'] and last_activity < cull_limit:
            app_log.info("Culling %s (inactive since %s)", user['name'], last_activity)
            req = HTTPRequest(url=url + '/users/%s/server' % user['name'],
                method='DELETE',
                headers=auth_header,
            )
            futures.append((user['name'], client.fetch(req)))
        elif user['server'] and last_activity > cull_limit:
            app_log.debug("Not culling %s (active since %s)", user['name'], last_activity)
    
    for (name, f) in futures:
        yield f
        app_log.debug("Finished culling %s", name) 

def get(self):

		self.site_section 	= "404"
		# self.error_msg		= "404 - page not found"

		print 
		app_log.info("PageNotFoundHandler.post / uri : %s ", pformat(self.request.uri ) )
		# pprint.pprint (self.request.uri )

		app_log.info("PageNotFoundHandler.post / self.is_user_connected : %s ", pformat(self.is_user_connected ) )
		# print self.is_user_connected

		app_log.warning("PageNotFoundHandler.post / request : \n %s ", pformat(self.request ) )
		# pprint.pprint (self.request )
		
		app_log.info("PageNotFoundHandler.post / request.arguments : \n %s ", pformat(self.request.arguments ) )
		# pprint.pprint( self.request.arguments )

		self.set_status(404)
		self.render("404.html",

			page_title  		= app_main_texts["main_title"],
			app_host			= self.request.host,

			site_section 		= self.site_section,
			
			error_msg 			= self.error_msg,
			
			user				= self.current_user,
			is_user_connected 	= self.is_user_connected,
			user_email			= self.user_email,
			user_auth_level		= self.user_auth_level,
			user_auth_level_dict = self.user_auth_level_dict
		)



### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### INDEX PAGE ##############################################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ### 

def test_put_user(self):
        client = yield self.auth_client()
        user = yield self.create_user(client)

        cases = [
            ("login", "foofoofoo"),
            ("email", "aaa@bbb.com"),
            ("is_admin", False),
            ("disabled", False),
            ("password", str(uuid.uuid4()))
        ]

        for i in range(1, len(cases)):
            for case in itertools.combinations(cases, i):
                body = dict(case)

                if 'disabled' in body:
                    log.info("Deleting user: %r", user['id'])
                    yield client.fetch(
                        self.get_url("/api/v1/user/{0}".format(user['id'])),
                        "DELETE"
                    )

                log.info("Send body: %r", body)
                response = yield client.fetch(
                    self.get_url("/api/v1/user/{0}".format(user['id'])),
                    "PUT", body
                )

                for k, v in body.items():
                    if k == 'password':
                        continue

                    self.assertIn(k, response.body)
                    self.assertEqual(v, response.body[k]) 

def test_put_errors(self):
        client = yield self.auth_client()
        user = yield self.create_user(client)

        cases = [
            ("login", False),
            ("login", [2, 3]),
            ("email", "@bbb.com"),
            ("email", {1: 2}),
            ("password", "123"),
            ("password", "1"),
            ("password", [1, '2']),
            ("password", {1: '2'}),
        ]

        for i in range(1, len(cases)):
            for case in itertools.combinations(cases, i):
                body = dict(case)

                with self.assertRaises(HTTPError) as err:
                    log.info("Body: %s", body)
                    yield client.fetch(
                        self.get_url("/api/v1/user/{0}".format(user['id'])),
                        "PUT", body
                    )

                self.assertEqual(err.exception.code, 400) 

def get(self):
		
		print
		app_log.info("RegisterHandler.get ... ")

		self.site_section = "register"

		# print "\nRegisterHandler.post / request : "
		# pprint.pprint (self.request )
		# print "\nRegisterHandler.post / request.arguments : "
		# pprint.pprint( self.request.arguments )

		# catch error message if any
		self.catch_error_message()

		next_url = self.get_argument('next', u'/')
		app_log.info("RegisterHandler.get / next_url : %s", next_url ) 
		# print next_url

		self.render('login_register.html',
			page_title  		= app_main_texts["main_title"],
			site_section		= self.site_section,
			app_host			= self.request.host,
			next_url 			= next_url,
			login_or_register 	= "register",
			error_msg			= self.error_msg,
			is_user_connected 	= self.is_user_connected

		) 

def get(self):

		app_log.info("InfosAPIdocHandler.get...")
		# app_log.info("InfosAPIdocHandler.get... \n : ", pformat(self.request.__dict__))
		
		self.site_section = "infos"

		example_item 	= self.application.coll_data.find()[0]
		example_item_id = str(example_item["_id"])
		
		example_spider 	= self.application.coll_spiders.find()[0]
		example_spider_id = str(example_spider["_id"])

		example_user 	= self.application.coll_users.find()[0]
		example_user_id = str(example_user["_id"])

		self.render(
			"api_doc.html",
			page_title 				= app_main_texts["main_title"],

			app_host				= self.request.host,
			site_section			= self.site_section,

			example_item_id			= example_item_id,
			example_spider_id		= example_spider_id,
			example_user_id			= example_user_id,

			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,
		) 

def get(self) : 
		
		print 
		app_log.info("DataModelEditHandler.get... " )

		self.site_section = "datamodel"

		# catch error message if any
		self.catch_error_message()


		# redirect if user doesn't have adapted auth level
		auth_level = self.user_auth_level_dict[self.site_section]
		self.redirect_user_if_not_authorized(auth_level, self.site_section)


		### retrieve datamodel from DB
		data_model_custom = list(self.application.coll_model.find({"field_class" : "custom"}))
		app_log.info("DataModelEditHandler.get / data_model_custom[0:2] : \n %s \n ...", pformat(data_model_custom[0:2])  )
		# pprint.pprint (data_model_custom)

		self.render(
			"datamodel_edit.html",
			page_title 			= app_main_texts["main_title"],
			site_section		= self.site_section,

			app_host			= self.request.host,

			field_types 		= DATAMODEL_FIELDS_TYPES,
			field_keep_vars	 	= DATAMODEL_FIELD_KEEP_VARS,
			field_open_vars	 	= DATAMODEL_FIELD_OPEN_VARS,
			
			datamodel_custom 	= data_model_custom,
			error_msg			= self.error_msg,
			
			user				= self.current_user,
			is_user_connected 	= self.is_user_connected,
			user_email			= self.user_email,
			user_auth_level		= self.user_auth_level,		
			user_auth_level_dict = self.user_auth_level_dict,
			) 

def check_docker_registry(self):
        """Check docker registry health"""
        app_log.info("Checking registry status")
        registry = self.settings["registry"]
        # we are only interested in getting a response from the registry, we
        # don't care if the image actually exists or not
        image_name = self.settings["image_prefix"] + "some-image-name:12345"
        await registry.get_image_manifest(
            *'/'.join(image_name.split('/')[-2:]).split(':', 1)
        )
        return True 

def authenticate(self, handler, data=None):
        code = handler.get_argument("code")
        http_client = AsyncHTTPClient()

        params = dict(
            client_id=self.client_id,
            client_secret=self.client_secret,
            grant_type='authorization_code',
            code=code,
            redirect_uri=self.get_callback_url(handler))

        data = urllib.parse.urlencode(
            params, doseq=True, encoding='utf-8', safe='=')

        url = self.token_url

        headers = {
            'Content-Type':
            'application/x-www-form-urlencoded; charset=UTF-8'
        }
        req = HTTPRequest(
            url,
            method="POST",
            headers=headers,
            body=data  # Body is required for a POST...
        )

        resp = await http_client.fetch(req)
        resp_json = json.loads(resp.body.decode('utf8', 'replace'))

        # app_log.info("Response %s", resp_json)
        access_token = resp_json['access_token']

        id_token = resp_json['id_token']
        decoded = jwt.decode(id_token, verify=False)

        userdict = {"name": decoded[self.username_claim]}
        userdict["auth_state"] = auth_state = {}
        auth_state['access_token'] = access_token
        # results in a decoded JWT for the user data
        auth_state['user'] = decoded

        return userdict 

def cull_idle(url, api_token, timeout):
    #last valid activity timestame
    cull_limit = datetime.datetime.utcnow() - datetime.timedelta(seconds=timeout)
    
    #get user list
    hub_api_authorization_header = { 'Authorization': 'token %s' % api_token}
    users_request = HTTPRequest(url=url + '/users', headers=hub_api_authorization_header )
    
    #run request tornado-asynchronously, extract user list (contains more information)
    resp = yield AsyncHTTPClient().fetch(users_request)
    all_users = json.loads(resp.body.decode('utf8', 'replace'))
    
    #build a bunch of (asynchronous) HTTP request futures...
    stop_notebook_futures = []
    servers_to_check = []
    dont_cull_these = set()
    for user in all_users:

        #extract last activity time, determine cullability of the server.
        last_activity = parse_date(user['last_activity'])
        should_cull = last_activity.replace(tzinfo=None)  < cull_limit.replace(tzinfo=None)
        user_name = user['name']
        app_log.debug("checking %s, last activity: %s, server: %s" % (user_name, last_activity, user['server']) )
        
        if not should_cull:
            dont_cull_these.add(user_name)
        
        #server should be culled:
        if user['server'] and should_cull:
            app_log.info("Culling %s (inactive since %s)", user_name, last_activity)
            stop_user_request = HTTPRequest(url=url + '/users/%s/server' % user_name,
                                            method='DELETE',
                                            headers=hub_api_authorization_header )
            stop_notebook_futures.append( (user_name, AsyncHTTPClient().fetch(stop_user_request)) )

        #Server status is None, which means actual status needs to be checked.
        if not user['server'] and should_cull:
            servers_to_check.append(user_name)

        #server should not be culled, just a log statement
        if user['server'] and not should_cull:
            app_log.info("Not culling %s (active since %s)", user['name'], last_activity)
            
    # Cull notebooks using normal API.
    for (user_name, cull_request) in stop_notebook_futures:
        try:
            yield cull_request #this line actually runs the api call to kill a server
        except HTTPError:
            #Due to a bug in Jupyterhub
            app_log.error("Something went wrong culling %s, will be manually killing it.", user_name)
            servers_to_check.append( user_name )
            continue
        app_log.info("Finished culling %s", user_name)
        
    for user_name in servers_to_check:
        if user_name not in dont_cull_these:
            yield manually_kill_server(user_name) 

def get (self ):
		
		print
		app_log.info("DataDatavizHandler.get ... : \n")

		self.site_section = "data"

		app_log.info("DataDatavizHandler.get / request : " )
		pprint.pprint (self.request )

		print
		app_log.info("DataDatavizHandler.get ... : ")
		app_log.info("... request.path : %s ", self.request.path )
		app_log.info("... request.uri  : %s ", self.request.uri )






		self.render(
			"dataviz.html",
			page_title 				= app_main_texts["main_title"],

			app_host				= self.request.host,
			site_section			= self.site_section,

			user					= self.current_user,
			is_user_connected 		= self.is_user_connected,
			user_email				= self.user_email,
			user_auth_level			= self.user_auth_level,
			user_auth_level_dict 	= self.user_auth_level_dict,


		)



# class DataScrapedViewOneHandler(BaseHandler):
# 	"""
# 	list all data scraped from db.data_scraped 
# 	"""
# 	@print_separate(APP_DEBUG)
# 	def get (self, spidername=None):
# 		self.redirect("/404")



### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### AJAX handler ############################################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###

# cf : https://stackoverflow.com/questions/23984586/reply-to-ajax-request-using-tornado
# cf : https://gist.github.com/mminer/5464753

# cf : https://github.com/richstoner/TornadoAJAXSample/blob/master/templates/index.html
# cf : https://github.com/richstoner/TornadoAJAXSample/blob/master/app.py 

def post(self):

		print 
		app_log.info("ContributorResetDataHandler.post ... " )

		app_log.info("ContributorResetDataHandler.post / request.arguments : \n %s ", pformat(self.request.arguments ) )

		# TO DO : form validation 

		### get reset choice + data validation
		spider_id	= self.get_argument("spider_id")
		app_log.info("ContributorResetDataHandler.post / spider_id : %s", spider_id )
		
		spider_oid = ObjectId(spider_id)

		is_reset	= self.get_argument("reset_data")
		app_log.info("ContributorResetDataHandler.post / is_reset : %s", is_reset )
		
		# reset collection here
		if is_reset == "true" :

			contributor	= self.application.coll_spiders.find_one({"_id": spider_oid })

			app_log.warning("ContributorResetDataHandler.post / DELETING DOCUMENTS IN COLL_DATA for spider_id : %s", spider_id )
			self.application.coll_data.delete_many({ "spider_id" : spider_id })
			
			# add warning message (as error for now)
			self.error_msg = self.add_error_message_to_slug( 
								error_string	= "all data from -%s- were erased" %(contributor["infos"]["name"]),
								args_to_delete 	= QUERY_SPIDER_BY_DEFAULT.keys()
								)

			# update scraper log
			self.update_spider_log(spider_id=spider_id, spider_oid=spider_oid, log_to_update="is_data_available", value=False)

			# redirect
			self.redirect("/contributors" + self.error_msg )

		else : 
			self.redirect("/contributors")





### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### DATA lists / edit handlers ##############################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###

### TO DO - after item pipeline 

def get(self):

		print
		app_log.info("ContributorResetDataHandler.get ..." )

		# catch error if any
		self.catch_error_message()

		self.site_section = "contributors"

		# redirect if user doesn't have adapted auth level
		auth_level = self.user_auth_level_dict[self.site_section]
		self.redirect_user_if_not_authorized(auth_level, self.site_section)


		spider_id = self.get_argument('spider_id', None )
		app_log.info("ContributorResetDataHandler.get / spider_id : %s", spider_id )

		spider_oid = ObjectId(spider_id)

		# spider exists ( edit form ) 
		if spider_id :
			
			try : 
				contributor	= self.application.coll_spiders.find_one({"_id": spider_oid })

				# redirect
				self.render(
					"contributor_reset_data.html",
					page_title  		= app_main_texts["main_title"],
					# site_section		= self.site_section, 

					app_host			= self.request.host,

					spider_id			= spider_id,
					contributor			= contributor,

					error_msg			= self.error_msg,
					
					user				= self.current_user,
					is_user_connected 	= self.is_user_connected,
					user_email			= self.user_email,
					user_auth_level		= self.user_auth_level,
					user_auth_level_dict = self.user_auth_level_dict,
					)
				
			except :
				app_log.warning("ContributorResetDataHandler.get --- !!! spider_id -%s- not found", spider_id ) 
				
				self.error_msg = self.add_error_message_to_slug( 
									error_string	= "there is no spider configuration with -%s- spider_id in the DB" %(str(spider_id)),
									args_to_delete 	= QUERY_SPIDER_BY_DEFAULT.keys()
									)
				self.redirect("/contributors" + self.error_msg)

		else :
			self.redirect("/contributors") 

def get(self):
		
		print
		app_log.warning("ContributorDeleteHandler.get / contributors :")

		# catch error if any
		self.catch_error_message()

		self.site_section = "contributors"

		# redirect if user doesn't have adapted auth level
		auth_level = self.user_auth_level_dict[self.site_section]
		self.redirect_user_if_not_authorized(auth_level, self.site_section)


		spider_id = self.get_argument('spider_id', None )
		app_log.info("ContributorDeleteHandler.get / spider_id : %s", spider_id )

		spider_oid = ObjectId(spider_id)

		# spider exists ( edit form ) 
		if spider_id :
			
			try : 
				contributor	= self.application.coll_spiders.find_one({"_id": spider_oid })
			
				self.render( "contributor_delete.html",
					
					page_title  		= app_main_texts["main_title"],
					# site_section		= self.site_section, 
					app_host			= self.request.host,

					spider_id			= spider_id,
					contributor			= contributor,

					error_msg			= self.error_msg,
					
					user				= self.current_user,
					is_user_connected 	= self.is_user_connected,
					user_email			= self.user_email,
					user_auth_level		= self.user_auth_level,
					user_auth_level_dict = self.user_auth_level_dict,
				)

			except :
				app_log.warning("ContributorDeleteHandler.get --- !!! spider_id -%s- not found", spider_id ) 
				
				self.error_msg = self.add_error_message_to_slug( 
									error_string	= "there is no spider configuration with -%s- spider_id in the DB" %(str(spider_id)),
									args_to_delete 	= QUERY_SPIDER_BY_DEFAULT.keys()
									)
				self.redirect("/contributors" + self.error_msg)


		else :
			self.redirect("/contributors") 

def get(self) : 

		print
		app_log.info("DataModelViewHandler.get... ")

		self.site_section = "datamodel"

		# catch error message if any
		self.catch_error_message()





		### retrieve datamodel from DB
		data_model_custom = list(self.application.coll_model.find({"field_class" : "custom"}).sort("field_name",1) )
		app_log.info("DataModelViewHandler.get / data_model_custom[0:2] : \n %s \n ...", pformat(data_model_custom[0:2]) )

		data_model_core = list(self.application.coll_model.find({"field_class" : "core"}).sort("field_name",1) )
		app_log.info("DataModelViewHandler.get / data_model_core[0:2] : \n %s \n ... ", pformat(data_model_core[0:2]) )





		### test printing object ID
		app_log.info("DataModelViewHandler.get / data_model_core[0] object_ID : %s ", str( data_model_core[0]["_id"]) )

		self.render(
			"datamodel_view.html",
			page_title 			= app_main_texts["main_title"],
			site_section		= self.site_section,
			
			app_host			= self.request.host,

			datamodel_custom 	= data_model_custom,
			datamodel_core 		= data_model_core,
			error_msg			= self.error_msg,
			
			user				= self.current_user,
			is_user_connected 	= self.is_user_connected,
			user_email			= self.user_email,
			user_auth_level		= self.user_auth_level,
			user_auth_level_dict = self.user_auth_level_dict,
		) 

def post(self):
		""" check if user exists in db and set cookie"""
		
		# self.check_xsrf_cookie()

		print 
		app_log.info("LoginHandler.post ...\n")
		
		app_log.info("LoginHandler.post / next_url : ")
		
		next_url = self.get_argument('next', '/')
		
		app_log.info( "%s - %s", next_url, type(next_url) )

		app_log.info("LoginHandler.post / request.arguments ... ")
		app_log.info( self.request.arguments )

		### get user from db
		user = self.get_user_from_db( self.get_argument("email") )
		app_log.info("LoginHandler.post / user : %s", user)
		# app_log.info( user )

		### TO DO : form validation 
		# form validation here....


		### check if user exists in db
		if user : 

			user_password	= user["password"]
			
			# check password 
			# TO DO : hash and/or decrypt password
			if self.get_argument("password") == user_password : 
				
				# set user
				self.set_current_user(user)

				# self.redirect("/")
				self.redirect( next_url )
			
			else : 
				# add error message and redirect if user wrote wrong password
				self.error_slug = self.add_error_message_to_slug("bad password or email mate ! no id stealing around here... mate !")
				self.redirect("/login/" + self.error_slug )
		
		else : 
			# add error message and redirect if no user registred in db
			# error_slug 		= u"?error=" + tornado.escape.url_escape("Login incorrect")
			self.error_slug = self.add_error_message_to_slug("incorrect login mate ! try again ")
			self.redirect("/login/" + self.error_slug) 

def get(self):
		
		app_log.info("WelcomeHandler.get... \n")

		self.site_section = "index"

		# catch error message if any
		self.catch_error_message()

		### count collections' documents
		counts = self.count_all_documents( 
					q_datamodel		= {"field_class" : "custom"},		 # query : just count custom fields
					q_contributors	= {"scraper_log.is_working" : True } # query : just count working and tested spiders
					) 
		app_log.info("WelcomeHandler.get / counts : \n  %s" , pformat(counts) )

		self.render( "index.html",

			page_title  		= app_main_texts["main_title"],
			site_section 		= self.site_section,
			counts 				= counts,
			
			app_host			= self.request.host,

			user				= self.current_user,
			is_user_connected 	= self.is_user_connected,
			user_email			= self.user_email,
			user_auth_level		= self.user_auth_level,
			user_auth_level_dict = self.user_auth_level_dict,

			error_msg			= self.error_msg,
		)

	# def write_error(self, status_code, **kwargs):
	# 	self.write("Gosh darnit, user! You caused a %d error." % status_code)



### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###
### LOGIN - LOGOUT - REGISTER HANDLERS ######################################################
### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###

# cf : https://guillaumevincent.github.io/2013/02/12/Basic-authentication-on-Tornado-with-a-decorator.html
# cf : http://tornado-web.blogspot.fr/2014/05/tornado-user-authentication-example.html 

def cleanup_builds(cls, kube, namespace, max_age):
        """Delete stopped build pods and build pods that have aged out"""
        builds = kube.list_namespaced_pod(
            namespace=namespace,
            label_selector='component=binderhub-build',
        ).items
        phases = defaultdict(int)
        app_log.debug("%i build pods", len(builds))
        now = datetime.datetime.now(tz=datetime.timezone.utc)
        start_cutoff = now - datetime.timedelta(seconds=max_age)
        deleted = 0
        for build in builds:
            phase = build.status.phase
            phases[phase] += 1
            annotations = build.metadata.annotations or {}
            repo = annotations.get("binder-repo", "unknown")
            delete = False
            if build.status.phase in {'Failed', 'Succeeded', 'Evicted'}:
                # log Deleting Failed build build-image-...
                # print(build.metadata)
                app_log.info(
                    "Deleting %s build %s (repo=%s)",
                    build.status.phase,
                    build.metadata.name,
                    repo,
                )
                delete = True
            else:
                # check age
                started = build.status.start_time
                if max_age and started and started < start_cutoff:
                    app_log.info(
                        "Deleting long-running build %s (repo=%s)",
                        build.metadata.name,
                        repo,
                    )
                    delete = True

            if delete:
                deleted += 1
                try:
                    kube.delete_namespaced_pod(
                        name=build.metadata.name,
                        namespace=namespace,
                        body=client.V1DeleteOptions(grace_period_seconds=0))
                except client.rest.ApiException as e:
                    if e.status == 404:
                        # Is ok, someone else has already deleted it
                        pass
                    else:
                        raise

        if deleted:
            app_log.info("Deleted %i/%i build pods", deleted, len(builds))
        app_log.debug("Build phase summary: %s", json.dumps(phases, sort_keys=True, indent=1))