Skip to content
GitLab
Commit 0cc6aa54 authored by M. Huang's avatar M. Huang
Browse files

rationalise auth verification and login, session. login before rigister

parent 8bceb8a2
syncker develop 2.8.3 2.8.2 2.8.1 2.8.0 2.7.2 2.7.1 2.7.0 ops-v1
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 322 additions and 140 deletions
+322 -140
Makefile_tests.mk
+ 11
- 0
View file @ 0cc6aa54
......@@ -73,3 +73,14 @@ test14:
test15:
$(PYTEST) $(OPT) --id test_threaded tests/serv/test_thread.py $(T)
U = http://127.0.0.1:9885/fdi/v0.16
P =
test16:
# clear up remain login info
curl -i -X GET $(U)/logout
curl -i -X GET $(U)/logout
rsp = `curl -i -X GET $(U)/`
print(rsp)
curl -i -X GET $(U)/logout
print(rsp)
fdi/_version.py
+ 3
- 2
View file @ 0cc6aa54
__version_info__ = (2, 6, 2)
__version_info__ = (2, 7, 0)
__version__ = '.'.join(map(str, __version_info__))
__revision__ = '2.4.0.1-1-g07a0c44'
# debug extra or missing pixel re png
# rationalise auth verification and login, session. login before rigister
# 2.6.2 debug extra or missing pixel re png
# 2.6.1 clean up dbg print
# 2.6.0 throw exception when csdb returns non-zero code.
# 2.5.3 # more msg from csdb resp content processing
......
fdi/httppool/__init__.py
+ 4
- 5
View file @ 0cc6aa54
......@@ -147,7 +147,7 @@ def setup_logging(level=LOGGING_NORMAL, extras=None, tofile=None):
logging.getLogger("urllib3").setLevel(extras)
return logging
# for those who cannot wait for create_app to set up logging
# for those who cannot wait for create_app to set up loggingb
logging = setup_logging()
########################################
......@@ -466,7 +466,7 @@ def create_app(config_object=None, level=None, logstream=None, preload=False):
# handlers for exceptions and some code
#add_errorhandlers(app)
# Do not redirect a URL ends with no spash to URL/
# Do not redirect a URL ends with no slash to URL/
app.url_map.strict_slashes = False
app.config['POOLS'] = {}
......@@ -517,7 +517,6 @@ if 1:
if logger.getEffectiveLevel() < logging.WARNING:
a = lls(request.view_args, 50)
# remove leading e.g. /fdi/v0.17
s = request.path.split(_BASEURL)
p = s[0] if len(s) == 1 else s[1] if s[0] == '' else request.path
method = request.method
......@@ -538,7 +537,7 @@ if 1:
from .model.user import auth
_c = (ctx(PM_S=PM_S, app=current_app, session=session,
request=request, auth=auth))
logger.debug(f"{_c}")
logger.debug(f"got sess pools. {_c}")
if 0:
GPL = PM_S.getMap().maps[0]
for pn, pu in pools.items():
......@@ -592,7 +591,7 @@ def ctx(PM_S, app, session, request, auth, users=False, **kwds):
headers = dict(request.headers)
cook = dict(request.cookies)
ses_user_id = session.get('user_id', 'No')
guser = getattr(g,'g_usr', 'No')
guser = getattr(getattr(g,'user',''), 'username', 'No')
m += (f" {_BLUE}Ses'{ses_user_id}' cnt{current_app.config['ACCESS']['usrcnt'][ses_user_id]} {headers.get('Authorization', '')[:12]}, gusr={guser} cookie.ses={cook.get('session', 'None')[-5:]}")
if users:
return m, c_user if c_user else auth_usr, ses_user_id, guser
......
fdi/httppool/auth.py
+ 0
- 2
View file @ 0cc6aa54
......@@ -9,8 +9,6 @@ from flask import (
)
from werkzeug.security import check_password_hash, generate_password_hash
from .db import get_db
user = Blueprint('user', __name__)
#
#
......
fdi/httppool/model/user.py
+ 117
- 68
View file @ 0cc6aa54
......@@ -167,61 +167,82 @@ def getUsers(pc):
if SESSION:
def set_user_session(username, pools=None, session=None, new=False, logger=logger):
""" set session user to username if username is different from the sessioin one.
""" set session user to username if username is different from the session one.
If the user id is not valid, `NameError` will be thrown.
"""
if session is None:
logger.debug('No session. Return.')
return
PM_S = PM_S_from_g(g)
_c = None
if logger.isEnabledFor(logging_DEBUG):
logger.debug(ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
if not username:
_c = ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth)
logger.debug(_c)
# if not username:
# logger.info(f'Null or blank is not a valid session user id.')
# return
ccu = current_app.config['USERS']
if username not in ccu:
logger.info(f'{username} is not a valid login id.')
return
if session.get('user_id', None) != username:
if getattr(getattr(g, 'user', ''), 'username', '') == username:
logger.info(f'{username} already the session user.')
return
else:
session['user_id'] = username
g.user = current_app.config['USERS'][username]
g.user = ccu[username]
# this will trigger session cookie-remaking
session.new = new
current_app.config['ACCESS']['usrcnt'][username] += 1
load_logged_in_user(username)
current_app.config['ACCESS']['usrcnt'][username] += 1
#load_logged_in_user(username)
if logger.isEnabledFor(logging_DEBUG):
m = (ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
logger.debug(f'END of set_usr_ses GPL {m}')
from ..session import set_session_pools
set_session_pools(session, PM_S, old_ctx=_c)
def load_logged_in_user(username):
""" put session user ID to g.user.
"""
logger = current_app.logger
if not SESSION:
if logger.isEnabledFor(logging_DEBUG):
logger.warning('Called with no SESSION')
return
# logger.debug(f'END of set_usr_ses GPL m')
PM_S = PM_S_from_g(g)
assert id(PM_S._GlobalPoolList.maps[0]) == id(pm_mod._PM_S._GlobalPoolList.maps[0])
if g is None or not hasattr(g, 'user') or g.user is None or g.user is None or g.user.username == username:
return
else:
g.user = username
# if user_id:
# if 'registered_pools' not in session:
# session['registered_pools'] = {}
# pools = session.get('registered_pools', {})
# # pools = current_app.config.get('POOLS', {}).get('user_id', {})
# else:
# pools = {}
# def load_logged_in_user(username):
# """ put session user ID to g.user.
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
_d = f"Updated g.usr {_YELLOW}{g.user}"
_c = (ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
logger.debug(f"{_BLUE}Load_Usr, {_d} {_c}")
# leave restoring the user's pools to session tokens.
# """
# logger = current_app.logger
# if not SESSION:
# if logger.isEnabledFor(logging_DEBUG):
# logger.warning('Called with no SESSION')
# return
# PM_S = PM_S_from_g(g)
# assert id(PM_S._GlobalPoolList.maps[0]) == id(pm_mod._PM_S._GlobalPoolList.maps[0])
# ccu = current_app.config['USERS']
# #if g is None or not hasattr(g, 'user') or g.user is None or g.user is None or g.user.username == username:
# try:
# if username not in ccu or g.user.username == username:
# logger.info(f'{username} is bad username or already the session user.')
# return
# except AttributeError:
# pass
# __import__("pdb").set_trace()
# g.user = ccu[username]
# # if user_id:
# # if 'registered_pools' not in session:
# # session['registered_pools'] = {}
# # pools = session.get('registered_pools', {})
# # # pools = current_app.config.get('POOLS', {}).get('user_id', {})
# # else:
# # pools = {}
# if SES_DBG and logger.isEnabledFor(logging_DEBUG):
# _d = f"Updated g.usr {_YELLOW}{g.user}"
# _c = (ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
# logger.debug(f"{_BLUE}Load_Usr, {_d} {_c}")
@user_api.after_app_request
def save_user(resp):
......@@ -234,7 +255,13 @@ if SESSION:
return resp
gu = getattr(g, 'user', None)
if not gu or 'user_id' not in session or gu.username == session['user_id']:
if not gu or 'user_id' not in session:
gu = None
session.pop('user_id', '')
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
logger.debug('Clear any user in sess and g.')
return resp
if gu.username == session['user_id']:
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
logger.debug('No need to save g.user.')
return resp
......@@ -268,8 +295,8 @@ def get_user_roles(user):
######################################
@user_api.route('/login/', methods=['GET', 'POST'])
@user_api.route('/login', methods=['GET', 'POST'])
@user_api.route('/login/', methods=['POST', 'GET'])
@user_api.route('/login', methods=['POST', 'GET'])
# @auth.login_required(role=['read_only', 'read_write'])
def login():
""" Logging in on the server.
......@@ -290,24 +317,27 @@ def login():
except (AttributeError, TypeError):
reqanm = reqaps = ''
if logger.isEnabledFor(logging_DEBUG):
msg = 'LOGIN meth=%s req_auth_nm= "%s"' % (
request.method, '*' * len(reqanm))
msg = 'LOGIN meth=%s req_auth_nm,ps= "%s" %s' % (
request.method, reqanm,'*' * len(reqaps))
logger.debug(msg)
from ..route.httppool_server import resp
logger.info(request.method)
if request.method == 'GET':
guser = getattr(g, 'user', None)
if guser is None and (request.authorization is None or not request.authorization.get('username', '')):
# guser = getattr(g, 'user', None)
# if 1: #guser is None and (request.authorization is None or not request.authorization.get('username', '')):
if request.authorization is None or \
not request.authorization.get('username', ''):
msg = 'Username and password please.'
if SESSION:
if LOGIN_TMPLT:
flash(msg)
return make_response(render_template(LOGIN_TMPLT))
return resp(401, 'Authentication needed.', msg, ts, req_auth=True)
# now request.authorization is None or len(request.authorization.get('username', '') == ''.get('username', '')) > 0
pass
rnm = request.authorization.get('username', '')
rpas = request.authorization.get('passwd', '')
elif request.method == 'POST':
rnm = request.form.get('username', None)
......@@ -327,7 +357,6 @@ def login():
rnm = rpas = None
else:
# still logged in
#__import__("pdb").set_trace()
# use User as the password
rnm, rpas = g.user.username, g.user
else:
......@@ -339,13 +368,17 @@ def login():
# still logged in XXX take new
rnm = request.authorization.get('username', None)
rpas = request.authorization.get('password', None)
if rnm is None:
# use User as the password
rnm, rpas = g.user.username, g.user
if logger.isEnabledFor(logging_DEBUG):
logger.debug(f'Request auth {rnm}')
vp = verify_password(rnm, rpas, check_session=True)
if vp not in (False, None):
msg = 'User %s logged-in %s.' % (vp, vp.roles)
msg = f'User {vp} logged-in {vp.roles}.'
if logger.isEnabledFor(logging_DEBUG):
logger.debug(msg)
# return redirect(url_for('pools.get_pools_url'))
......@@ -429,22 +462,22 @@ def verify_password(username, password, check_session=True):
`check_session`=`True` ('u/k' means unknown)
=========== ============= ======= ========== ========= ==================
state `session` `g` username password action
=========== ============= ======= ========== ========= ==================
no Session no 'user_id' not empty valid new session, r/t new u
no Session no 'user_id' not empty invalid login, r/t `False`
no Session no 'user_id' '' r/t None
no Session no 'user_id' None, u/k login, r/t `False`
no SESSION not enabled not empty cleartext approve
In session w/ 'user_id' '' different valid new session, r/t new u
In session w/ 'user_id' None empty -- no session, r/t no u
.. not same
In session w/ 'user_id' not empty invalid login, return `False`
In session w/ 'user_id' user == user valid login, return same user
In session w/ 'user_id' user None "" login, return `False`
.. u/k
=========== ============= ======= ========== ========= ==================
= =========== ============= ======= ========== ========= =======================
n state `session` `g` username password action
= =========== ============= ======= ========== ========= =======================
0 no Session no 'user_id' not empty valid new session, r/t new u
1 no Session no 'user_id' not empty invalid login, r/t `False`
2 no Session no 'user_id' '' r/t None
3 no Session no 'user_id' None, u/k login, r/t `False`
4 no SESSION not enabled not empty cleartext approve
5 In session w/ 'user_id' '' None diff knwn valid new ses, r/t new u
6 In session w/ 'user_id' '' None empty -- no session, r/t none
7 In session w/ 'user_id' -- not empty invalid login, return `False`
8 In session w/ 'user_id' user diff knwn valid clos ol sess,new session, r/t new u
9 In session w/ 'user_id' user == user valid same sess, rt same user
A In session w/ 'user_id' user None "" login, return `False`
B unknown same
= =========== ============= ======= ========== ========= =======================
`check_session`=`False`
......@@ -480,7 +513,8 @@ def verify_password(username, password, check_session=True):
'check' if check_session else 'nochk',
session.get('user_id','NO SESS USR') if SESSION else 'noSess',
getattr(g, 'user','NO g USR')))
logger.debug(ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
_c = ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth)
logger.debug(_c)
if check_session:
if SESSION:
......@@ -495,19 +529,24 @@ def verify_password(username, password, check_session=True):
# first check if the username is actually unchanged and valid
if newu is not None and ((password is user) or newu.is_correct_password(password)):
if gname == username:
# case 9
# username is from header AUTHR..
if logger.isEnabledFor(logging_DEBUG):
pools = dict((p, o._poolurl)
for p, o in PM_S.getMap().items())
logger.debug(f"Same session {pools}.")
set_user_session(
set_user_session(
username, session=session, logger=logger)
return user
#################
else:
# case 5 case 8
# headers do not agree with cookies token. take header's
from ..session import close_session
close_session(session, PM_S, _c)
if logger.isEnabledFor(logging_INFO):
logger.info(f"New session {username}")
set_user_session(
username, session=session, new=True, logger=logger)
return newu
......@@ -515,6 +554,7 @@ def verify_password(username, password, check_session=True):
if logger.isEnabledFor(logging_DEBUG):
logger.debug(
f"Unknown {username} or Null or anonymous user, or new user '{username}' has invalid password.")
# case 7, A, B
return False
#################
else:
......@@ -533,20 +573,24 @@ def verify_password(username, password, check_session=True):
newu = current_app.config['USERS'].get(username, None)
if newu is None:
# case 6
if logger.isEnabledFor(logging_DEBUG):
logger.debug(f"{stat}Unknown user {username}")
return False
#################
if newu.is_correct_password(password):
# case 5
if logger.isEnabledFor(logging_INFO):
logger.info(
f"{stat}Approved user {username}. Start session.")
set_user_session(newu.username, pools=None,
session=session, new=True, logger=logger)
return newu
#################
else:
# case 7
if logger.isEnabledFor(logging_DEBUG):
logger.debug(
f"{stat}new user '{username}' has invalid password.")
......@@ -556,11 +600,13 @@ def verify_password(username, password, check_session=True):
# SESSION not enabled. Use clear text passwd
newu = current_app.config['USERS'].get(username, None)
if newu and newu.is_correct_password(password):
# case 0
if logger.isEnabledFor(logging_INFO):
logger.info(f'Approved new user {username} w/o session')
return newu
#################
else:
# case 1, 3
if logger.isEnabledFor(logging_DEBUG):
logger.debug(
f"Null or anonymous user, or new user '{username}' has invalid password.")
......@@ -569,17 +615,20 @@ def verify_password(username, password, check_session=True):
else:
# check_session is False. called by login to check formed name/pass
if username == '':
# case 2
if logger.isEnabledFor(logging_DEBUG):
logger.debug('LOGIN: check anon')
return None
#################
newu = current_app.config['USERS'].get(username, None)
if newu is None:
# case 3
if logger.isEnabledFor(logging_DEBUG):
logger.debug(f"LOGIN: Unknown user {username}")
return False
#################
if newu.is_correct_password(password):
# case 0
if logger.isEnabledFor(logging_INFO):
logger.info('LOGIN Approved {username}')
return newu
......
fdi/httppool/route/httppool_server.py
+ 3
- 1
View file @ 0cc6aa54
......@@ -66,7 +66,7 @@ PM_S = None
def resp(code, result, msg, ts, serialize_out=False,
ctype='application/json',
length=80, req_auth=False, extra_headers=None):
length=200, req_auth=False, extra_headers=None):
"""
Make response.
......@@ -90,6 +90,8 @@ def resp(code, result, msg, ts, serialize_out=False,
w = serialize({"result": result, "msg": msg, "time": ts})
else:
w = result
if 'Only6' in w:
__import__("pdb").set_trace()
if logger.isEnabledFor(logging_DEBUG):
logger.debug('resp result:' + lls(w, length))
......
fdi/httppool/route/pools.py
+ 25
- 11
View file @ 0cc6aa54
......@@ -31,6 +31,20 @@ from ...utils.common import (lls,
logging_DEBUG,
trbk
)
from ...utils.colortext import (ctext,
_CYAN,
_GREEN,
_HIWHITE,
_WHITE_RED,
_YELLOW,
_MAGENTA,
_BLUE_DIM,
_BLUE,
_RED,
_BLACK_RED,
_RESET
)
from flask import g, Blueprint, jsonify, request, current_app, url_for, abort, session
from werkzeug.exceptions import HTTPException
# from flasgger import swag_from
......@@ -126,10 +140,10 @@ def get_pools_url():
guser = getattr(g, 'user', None)
name = getattr(guser, 'username', '')
roles = getattr(guser, 'roles', '')
loggedin = guser is not None and getattr(guser, 'logged_in', False)
loggedin = guser is not None
u += '%s: logged in as %s.' % (name, str(roles)) if loggedin else "not logged in."
if logger.isEnabledFor(logging_DEBUG):
logger.debug(u)
logger.debug(f"{_YELLOW}{u}")
msg = '%d pools found. Version: fdi %s, docker %s, HTTPPool server %s, %s' % (
len(res), __revision__, dvers if dvers else 'None',
......@@ -466,8 +480,8 @@ def unregister_pools(poolnames=None):
@ pools_api.route('/pools/wipe_all', methods=['DELETE'])
# @ pools_api.route('/pools/wipe_all/', methods=['DELETE'])
#@ auth.login_required(role=['all_doer'])
@ pools_api.route('/pools/wipe_all/', methods=['DELETE'])
@ auth.login_required(role=['all_doer'])
def wipe_all():
""" Remove contents of all pools.
......@@ -532,14 +546,14 @@ def wipe_pools(poolnames, usr):
return good, notgood
######################################
#### /{pool} /{pool}/ GET ####
######################################
###########################################
#### /{pool} /{pool}/ GET, PUT ####
###########################################
@ pools_api.route('/<string:pool>/', methods=['GET'])
@ pools_api.route('/<string:pool>', methods=['GET'])
# @ auth.login_required(role=['read_only'])
@ auth.login_required(role=['read_only', 'read_write'])
def get_pool(pool):
""" Get information of the given pool.
......@@ -666,7 +680,7 @@ def register_pool(poolname, usr, poolurl=None):
return code, result, msg
################################################
#### {pool}/register PUT ####
#### {pool} PUT ####
################################################
......@@ -688,7 +702,7 @@ def register(pool):
PM_S = PM_S_from_g(g)
assert id(PM_S._GlobalPoolList.maps[0]) == id(pm_mod._PM_S._GlobalPoolList.maps[0])
logger.debug(ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth))
if logger.isEnabledFor(logging_DEBUG):
logger.debug(f"Registering pool @ {pool}")
......@@ -769,7 +783,7 @@ def unregister_pool(pool):
######################################
# @ pools_api.route('/<string:pool>/hk', methods=['GET'])
@ pools_api.route('/<string:pool>/hk', methods=['GET'])
@ pools_api.route('/<string:pool>/hk/', methods=['GET'])
@ auth.login_required(role=['read_only', 'read_write'])
def hk(pool):
......
fdi/httppool/session.py
+ 62
- 28
View file @ 0cc6aa54
......@@ -2,7 +2,9 @@
from ..utils.getconfig import getConfig
from ..utils.common import lls
from ..pal.poolmanager import PM_S_from_g
from ..utils.colortext import (ctext,
_CYAN,
_GREEN,
......@@ -50,6 +52,42 @@ METHODS = ("GET",)
# 0 means disabled
MAX_RETRIES = 0
def set_session_pools(session, PM_S, old_ctx):
logger = current_app.logger
if not SESSION:
if logger.isEnabledFor(logging_DEBUG):
logger.debug('Called with no SESSION')
return
GPL = PM_S._GlobalPoolList
pools = session.get('registered_pools', {})
nfound = 0
for pn, pu in pools.items():
if 0: #'ol_1' in pn:
__import__("pdb").set_trace()
if pn not in GPL:
logger.info(f'{_RED}{pn} is not found in GPL.')
#code, po, msg = register_pool(
# pn, current_app.config['USERS'][username], poolurl=pu)
#assert po is GPL[pn]
else:
nfound += 1
#assert pn in GPL
gpu = GPL[pn]._poolurl
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
logger.debug(f'GPL.p/u {lls(gpu,20)}{"==" if gpu == pu else "!="}cookie p/u {lls(pu,20)}.')
logger.info(f"{nfound} of {len(pools)} pools in cookie found in GPL.")
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
from . import ctx
from .model.user import auth
_c1 = ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth)
logger.debug(f"{_BLUE}Load_Ses {nfound} pools, {'ctx unchanged.' if old_ctx == _c1 else _c1}")
return
@session_api.before_app_request
def b4req_session():
""" Every session keeps record of what pools it had
......@@ -69,7 +107,6 @@ def b4req_session():
logger.warning('Called with no SESSION')
return
pools = session.get('registered_pools', {})
GPL = PM_S._GlobalPoolList
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
......@@ -78,24 +115,31 @@ def b4req_session():
_c = (ctx(PM_S=PM_S, app=current_app, session=session,
request=request, auth=auth))
logger.debug(f"{_c}")
for pn, pu in pools.items():
if 0: #'ol_1' in pn:
__import__("pdb").set_trace()
#
if pn not in GPL:
logger.info(f'{_RED}{pn} is not found in GPL.')
#code, po, msg = register_pool(
# pn, current_app.config['USERS'][username], poolurl=pu)
#assert po is GPL[pn]
else:
#assert pn in GPL
gpu = GPL[pn]._poolurl
logger.info(f'GPL.purl {gpu}{"==" if gpu == pu else "!="}cookie {pu}.')
def close_session(session, PM_S, old_ctx):
logger = current_app.logger
if not SESSION:
if logger.isEnabledFor(logging_DEBUG):
logger.debug('Called with no SESSION')
return resp
GPL = PM_S.getMap().maps[0]
_d = {} if len(GPL) == 0 else dict((pn, x._poolurl) for pn, x in GPL.items())
session['registered_pools'] = _d
session.modified = True
_m = ''
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
_c1 = ctx(PM_S=PM_S, app=current_app, session=session, request=request, auth=auth)
logger.debug(f"{_BLUE}Load_Ses, {'ctx unchanged.' if _c == _c1 else _c1}")
return
from . import ctx
from .model.user import auth
_c2 = ctx(PM_S=PM_S, app=current_app, session=session,
request=request, auth=auth)
_m = 'ctx unchanged.' if old_ctx == _c2 else _c2
logger.debug(f"{_YELLOW}Updated SesPools {' '.join(session['registered_pools'])}. {_m}")
@session_api.after_app_request
def aftreq_session(resp):
......@@ -108,7 +152,7 @@ def aftreq_session(resp):
return resp
PM_S = PM_S_from_g(g)
_c = None
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
from . import ctx
from .model.user import auth
......@@ -116,17 +160,7 @@ def aftreq_session(resp):
request=request, auth=auth)
logger.debug(_c)
GPL = PM_S.getMap().maps[0]
_d = {} if len(GPL) == 0 else dict((pn, x._poolurl) for pn, x in GPL.items())
session['registered_pools'] = _d
session.modified = True
_m = ''
if SES_DBG and logger.isEnabledFor(logging_DEBUG):
_c2 = ctx(PM_S=PM_S, app=current_app, session=session,
request=request, auth=auth)
_m = 'ctx unchanged.' if _c == _c2 else _c2
logger.debug(f"{_YELLOW}Updated SesPools {' '.join(session['registered_pools'])}. {_m}")
close_session(session, PM_S, _c)
return resp
......
fdi/pal/poolmanager.py
+ 28
- 3
View file @ 0cc6aa54
......@@ -167,7 +167,7 @@ def get_secondary_poolurl(poolurl, poolhint='csdb:,,'):
return poolurl, secondary_poolurl, last_frag, schm
def remoteRegister(pool):
def remoteRegister(pool, makenew=False):
""" if registered a pool's auth and client will be used.
If the pool has a `secondary_poolurl` property, after '/' is
......@@ -210,8 +210,33 @@ def remoteRegister(pool):
if pool.auth:
pool.client.auth = pool.auth
from ..pns.fdi_requests import put_on_server
# try logging in
from ..pns.fdi_requests import put_on_server, post_to_server
from ..httppool.model.user import SESSION
# if SESSION:
# from ..httppool import g, app
# with app.app_context():
# gvar = getattr(g, 'user', None)
# try:
# # __import__("pdb").set_trace()
# if gvar is None:
# not logged in
try:
# do not serialize this
authd = {'username':poolo.auth.username,
'password':poolo.auth.password}
code, res, msg = post_to_server(authd,
'urn:::0', poolurl, 'login',
auth=poolo.auth, client=poolo.client,
no_serial=True)
except (ConnectionError, NewConnectionError) as e:
res, msg = 'FAILED', str(e)
logger.error(poolurl + ' ' + lls(msg, 220))
raise
try:
res, msg = put_on_server(
'urn:::0', poolurl, 'register_pool',
auth=poolo.auth, client=poolo.client)
......@@ -817,7 +842,7 @@ Pools registered are kept as long as the last reference remains. When the last i
if need_to_reg_save:
# remote register
if schm in ('http', 'https', 'csdb'):
res, msg = remoteRegister(pool)
res, msg = remoteRegister(pool, makenew=makenew)
# print(getweakrefs(p), id(p), '////')
# If the pool is a client pool, it is this pool that goes into
......
fdi/pal/productstorage.py
+ 1
- 1
View file @ 0cc6aa54
......@@ -317,7 +317,7 @@ class ProductStorage(object):
if poolname not in self._pools:
msg = 'pool ' + poolname + ' not found'
logger.error(msg)
raise ValueError(msg)
raise NameError(msg)
return self._pools[poolname]
def getWritablePool(self, obj=False):
......
fdi/pns/fdi_requests.py
+ 5
- 3
View file @ 0cc6aa54
......@@ -136,7 +136,7 @@ def urn2fdiurl(urn, poolurl, contents='product', method='GET'):
method:
'GET' compo for retrieving product or hk or classes, urns, tags,
'POST' compo for uploading product
'POST' compo for uploading product, and login
'PUT' for registering pool
'DELETE' compo for removing product or unregistering pool
......@@ -192,6 +192,8 @@ def urn2fdiurl(urn, poolurl, contents='product', method='GET'):
elif method == 'POST':
if contents == 'product':
ret = baseurl + poolname + '/'
elif contents == 'login':
return baseurl + '/' + contents
elif contents.split('__', 1)[0] in WebAPI:
# append a '/' for flask
ret = poolurl + '/api/' + contents.split('__', 1)[0] + '/'
......@@ -311,7 +313,7 @@ def post_to_server(data, urn, poolurl, contents='product', headers=None,
print('POST API: ' + api + ' | ' + lls(data, 90))
aheaders = auth_headers(auth.username, auth.password)
if headers :
if headers:
aheaders.update(headers)
headers = aheaders
......@@ -443,7 +445,7 @@ def put_on_server(urn, poolurl, contents='pool', headers=None, result_only=False
print('client session cookies', list(client.cookies))
aheaders = auth_headers(auth.username, auth.password)
if headers :
if headers:
aheaders.update(headers)
headers = aheaders
......
fdi/utils/tofits.py
+ 50
- 11
View file @ 0cc6aa54
......@@ -20,6 +20,7 @@ from .common import lls
import os
from collections.abc import Sequence
import io
from filelock import Timeout, FileLock
import copy
import datetime
import itertools
......@@ -164,7 +165,7 @@ def toFits(data, file='', headers_only=False, **kwds):
zim = data.zInfo['metadata']
if 'aId' in zim:
x = value2parameter('aId', value, zim)
if data.meta['aId'] != x:
if data.meta.get('aId', None) != x:
data.meta['aId'] = x
hdul[0].header['aId'] = (value, desc)
else:
......@@ -590,20 +591,28 @@ def write_to_file(p, fn, dct=None, ignore_type_error=False, this_fits=None, inde
try:
if is_Fits(p):
with open(sp, 'wb') as prodf:
prodf.write(p)
lock = FileLock(f'/tmp/{sp}.lock')
with lock:
with open(sp, 'wb') as prodf:
prodf.write(p)
elif issubclass(p.__class__, HDUList):
with open(sp, 'wb') as prodf:
p.writeto(prodf)
lock = FileLock(f'/tmp/{sp}.lock')
with lock:
with open(sp, 'wb') as prodf:
p.writeto(prodf)
else:
with open(sp, 'w+') as prodf:
prodf.write(serialize(p, indent=indent))
lock = FileLock(f'/tmp/{sp}.lock')
with lock:
with open(sp, 'w+') as prodf:
prodf.write(serialize(p, indent=indent))
# logger.info(f'Cannot save {p.__class__} to FITS.')
# cmd line
if _p:
_sp = os.path.splitext(sp)[0] + '.fit'
with open(_sp, 'wb') as prodf:
prodf.write(_p)
lock = FileLock(f'/tmp/{_sp}.lock')
with lock:
with open(_sp, 'wb') as prodf:
prodf.write(_p)
except TypeError as e:
if ignore_type_error:
......@@ -653,5 +662,35 @@ def send_samp(filepath, client, **kwds):
if __name__ == '__main__':
# test_fits_kw(fits_data())
main()
import logging
# create logger
logger = logging.getLogger()
logging.basicConfig(stream=sys.stdout,
format='%(asctime)s -%(levelname)4s'
' -[%(filename)s:%(lineno)3s'
' -%(funcName)10s()] - %(message)s',
datefmt="%Y%m%d %H:%M:%S")
logger.setLevel(logging.DEBUG)
# get commandline options. They may override envirionment variables
args, extra_args, remainings = get_args(defaults=CMDLINE_defaults,
extra_options=CMDLINE_extra_options,
cmdline=sys.argv[1:])
# use commandline defaults to override current default
df = {}.get if defaults is None else defaults.get
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog=''
)
with open('rb') as f:
buf = f.read()
from svom.products.loadfits_svom import loadfits_svom
if is_Fits(buf):
p = loadfits_svom
exit(main())
tests/serv/test_httpclientpool.py
+ 11
- 4
View file @ 0cc6aa54
......@@ -16,7 +16,7 @@ from fdi.pal.query import MetaQuery
from fdi.pal.poolmanager import PoolManager, DEFAULT_MEM_POOL, PM_S
from fdi.pal.httpclientpool import HttpClientPool
from fdi.pns.fdi_requests import safe_client, urn2fdiurl, parse_poolurl
from fdi.utils.common import fullname
from fdi.utils.common import fullname, trbk
import networkx as nx
......@@ -349,9 +349,16 @@ def test_CRUD_product_by_client(get_PS_for_CRUD):
str(pstore.getPools())
logger.info('Access a non-existing pool and trigger an Error.')
with pytest.raises(ValueError):
pstore.getPool(poolid + 'NON_EXISTS ') is None
return
if 0:
with pytest.raises(NameError):
pstore.getPool(poolid + '_NON_EXISTS') is None
else:
try:
pstore.getPool(poolid + '_NON_EXISTS') is None
except Exception as e:
print(f"Expected exception from server: {e}, traceback {trbk(e)}")
assert 'not found' in str(e)
def est_webapi_backup_restore(server):
"""
......
tests/test_pal.py
+ 2
- 1
View file @ 0cc6aa54
......@@ -1211,7 +1211,8 @@ def backup_restore(ps, client, auth):
lst = p2.restore(tar2)
# two pools are the same
assert deepcmp(hk1, p2.readHK()) is None
p2hk = p2.readHK()
assert deepcmp(hk1, p2hk) is None, print(hk1, p2hk)
# @pytest.fixture(scope='function')
# def mkStorage_1(make_remote_storage):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment