From 9a469732f21048aee305045cb1196f79311f7554 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Tue, 11 Jul 2023 23:03:59 -0400 Subject: [PATCH 1/7] [_463] simple_client Session and Connection These aren't replacements for irods.session.iRODSSession and irods.connection.Connection, rather they are being introduced at this time as an experiment and try to acquaint ourselves (and users of the PRC) to the ways in which things may be evolving. --- irods/simple_client/__init__.py | 872 ++++++++++++++++++++++++++++++++ 1 file changed, 872 insertions(+) create mode 100644 irods/simple_client/__init__.py diff --git a/irods/simple_client/__init__.py b/irods/simple_client/__init__.py new file mode 100644 index 00000000..34e700ff --- /dev/null +++ b/irods/simple_client/__init__.py @@ -0,0 +1,872 @@ +from __future__ import print_function +from __future__ import absolute_import + +import atexit +import ast +from ast import literal_eval as safe_eval +import copy +import datetime +import errno +import hashlib +import json +import logging +import os +import socket +import struct +import six +import os +import ssl +import irods.password_obfuscation as obf +from irods import MAX_NAME_LEN +import re + +from irods.password_obfuscation import decode +from irods.account import iRODSAccount +from irods import NATIVE_AUTH_SCHEME, PAM_AUTH_SCHEME +from irods.query import Query + +logger = logging.getLogger(__name__) + +PAM_PW_ESC_PATTERN = re.compile(r'([@=&;])') + +class NonAnonymousLoginWithoutPassword(RuntimeError): pass + +class Session(object): + + def __enter__(self): + self.pool.get_connection() + return self + + def __exit__(self,*x): + self.cleanup() + return + + def clone(self): + other = copy.copy(self) + other.account = copy.copy(self.account) + other.conn = None + return other + + def cleanup(self): + self.conn = None + + @property + def connection(self): + return self.pool.get_connection() # -> sets and returns self.conn + + @property + def server_version(self): + return self.connection.server_version + + # Provide a pool property so that we can act like an old-style iRODSSession + + @property + def pool(self): + class _: + @staticmethod + def get_connection(): + if self.conn is None: + self.conn = Connection(self.account) + return self.conn + return _() + + def __init__(self, configure = True, **kwargs): + self._env_file = '' + self._auth_file = '' + self.account = None + self.conn = None + if configure: + self.account = self.configure(**kwargs) + + def _configure_account(self, **kwargs): + + try: + env_file = kwargs['irods_env_file'] + + except KeyError: + # For backward compatibility + for key in ['host', 'port', 'authentication_scheme']: + if key in kwargs: + kwargs['irods_{}'.format(key)] = kwargs.pop(key) + + for key in ['user', 'zone']: + if key in kwargs: + kwargs['irods_{}_name'.format(key)] = kwargs.pop(key) + + return iRODSAccount(**kwargs) + + # Get credentials from irods environment file + creds = self.get_irods_env(env_file, session_ = self) + + # Update with new keywords arguments only + creds.update((key, value) for key, value in kwargs.items() if key not in creds) + + # Get auth scheme + try: + auth_scheme = creds['irods_authentication_scheme'] + except KeyError: + # default + auth_scheme = 'native' + + if auth_scheme.lower() == PAM_AUTH_SCHEME: + if 'password' in creds: + return iRODSAccount(**creds) + else: + # password will be from irodsA file therefore use native login + creds['irods_authentication_scheme'] = NATIVE_AUTH_SCHEME + elif auth_scheme != 'native': + return iRODSAccount(**creds) + + # Native auth, try to unscramble password + try: + creds['irods_authentication_uid'] = kwargs['irods_authentication_uid'] + except KeyError: + pass + + missing_file_path = [] + error_args = [] + pw = creds['password'] = self.get_irods_password(session_ = self, file_path_if_not_found = missing_file_path, **creds) + if not pw and creds.get('irods_user_name') != 'anonymous': + if missing_file_path: + error_args += ["Authentication file not found at {!r}".format(missing_file_path[0])] + raise NonAnonymousLoginWithoutPassword(*error_args) + + return iRODSAccount(**creds) + + def configure(self, **kwargs): + account = self.account + if not account: + account = self._configure_account(**kwargs) + return account + + def query(self, *args): + return Query(self, *args) + + @property + def username(self): + return self.account.client_user + + @property + def zone(self): + return self.account.client_zone + + @property + def host(self): + return self.account.host + + @host.setter + def host(self,host_name): + self.account.host = host_name + + @property + def port(self): + return self.account.port + + ''' + @property + def server_version(self): + try: + reported_vsn = os.environ.get("PYTHON_IRODSCLIENT_REPORTED_SERVER_VERSION","") + return tuple(ast.literal_eval(reported_vsn)) + except SyntaxError: # environment variable was malformed, empty, or unset + return self.__server_version() + + def __server_version(self): + try: + conn = next(iter(self.pool.active)) + return conn.server_version + except StopIteration: + conn = self.pool.get_connection() + version = conn.server_version + conn.release() + return version + + @property + def pam_pw_negotiated(self): + self.pool.account.store_pw = [] + conn = self.pool.get_connection() + pw = getattr(self.pool.account,'store_pw',[]) + delattr( self.pool.account, 'store_pw') + conn.release() + return pw + +''' + + @property + def default_resource(self): + return self.account.default_resource + + @default_resource.setter + def default_resource(self, name): + self.account.default_resource = name + + @property + def connection_timeout(self): + return self.connection_timeout + + @staticmethod + def get_irods_password_file(): + try: + return os.environ['IRODS_AUTHENTICATION_FILE'] + except KeyError: + return os.path.expanduser('~/.irods/.irodsA') + + @staticmethod + def get_irods_env(env_file, session_ = None): + try: + with open(env_file, 'rt') as f: + j = json.load(f) + if session_ is not None: + session_._env_file = env_file + return j + except IOError: + logger.debug("Could not open file {}".format(env_file)) + return {} + + @staticmethod + def get_irods_password(session_ = None, file_path_if_not_found = (), **kwargs): + path_memo = [] + try: + irods_auth_file = kwargs['irods_authentication_file'] + except KeyError: + irods_auth_file = Session.get_irods_password_file() + + try: + uid = kwargs['irods_authentication_uid'] + except KeyError: + uid = None + + _retval = '' + + try: + with open(irods_auth_file, 'r') as f: + _retval = decode(f.read().rstrip('\n'), uid) + return _retval + except IOError as exc: + if exc.errno != errno.ENOENT: + raise # Auth file exists but can't be read + path_memo = [ irods_auth_file ] + return '' # No auth file (as with anonymous user) + finally: + if isinstance(file_path_if_not_found, list) and path_memo: + file_path_if_not_found[:] = path_memo + if session_ is not None and _retval: + session_._auth_file = irods_auth_file + +#------------------------------------------------------------------------------------------ + + +from irods.message import ( + iRODSMessage, StartupPack, AuthResponse, AuthChallenge, AuthPluginOut, + OpenedDataObjRequest, FileSeekResponse, StringStringMap, VersionResponse, + PluginAuthMessage, ClientServerNegotiation, Error, GetTempPasswordOut) +from irods.exception import (get_exception_by_code, NetworkException, nominal_code) +from irods.message import (PamAuthRequest, PamAuthRequestOut) + + + +ALLOW_PAM_LONG_TOKENS = True # True to fix [#279] +# Message to be logged when the connection +# destructor is called. Used in a unit test +DESTRUCTOR_MSG = "connection __del__() called" + +from irods import ( + MAX_PASSWORD_LENGTH, RESPONSE_LEN, + AUTH_SCHEME_KEY, AUTH_USER_KEY, AUTH_PWD_KEY, AUTH_TTL_KEY, + NATIVE_AUTH_SCHEME, + GSI_AUTH_PLUGIN, GSI_AUTH_SCHEME, GSI_OID, + PAM_AUTH_SCHEME) +from irods.client_server_negotiation import ( + perform_negotiation, + validate_policy, + REQUEST_NEGOTIATION, + REQUIRE_TCP, + FAILURE, + USE_SSL, + CS_NEG_RESULT_KW) +from irods.api_number import api_number + +class Connection(object): + + DISALLOWING_PAM_PLAINTEXT = True + + def __enter__(self): + return self + + def __exit__(self,*_): + return + + def cleanup(self): + self.conn = None + def __init__(self, account): + + self.application_name = "PRC_mk_II" + self.socket = None + self.account = account + self._client_signature = None + self._server_version = self._connect() + self._disconnected = False + + scheme = self.account.authentication_scheme + + if scheme == NATIVE_AUTH_SCHEME: + self._login_native() + elif scheme == GSI_AUTH_SCHEME: + self.client_ctx = None + self._login_gsi() + elif scheme == PAM_AUTH_SCHEME: + self._login_pam() + else: + raise ValueError("Unknown authentication scheme %s" % scheme) + self.create_time = datetime.datetime.now() + self.last_used_time = self.create_time + + @property + def server_version(self): + detected = tuple(int(x) for x in self._server_version.relVersion.replace('rods', '').split('.')) + return (safe_eval(os.environ.get('IRODS_SERVER_VERSION','()')) + or detected) + @property + def client_signature(self): + return self._client_signature + + def __del__(self): + self.disconnect() + logger.debug(DESTRUCTOR_MSG) + + def send(self, message): + string = message.pack() + + logger.debug(string) + try: + self.socket.sendall(string) + except: + logger.error( + "Unable to send message. " + + "Connection to remote host may have closed. " + + "Releasing connection from pool." + ) + #self.release(True) + raise NetworkException("Unable to send message") + + def recv(self, into_buffer = None + , return_message = () + , acceptable_errors = ()): + acceptable_codes = set(nominal_code(e) for e in acceptable_errors) + try: + if into_buffer is None: + msg = iRODSMessage.recv(self.socket) + else: + msg = iRODSMessage.recv_into(self.socket, into_buffer) + except (socket.error, socket.timeout) as e: + # If _recv_message_in_len() fails in recv() or recv_into(), + # it will throw a socket.error exception. The exception is + # caught here, a critical message is logged, and is wrapped + # in a NetworkException with a more user friendly message + logger.critical(e) + logger.error("Could not receive server response") + #self.release(True) + raise NetworkException("Could not receive server response") + if isinstance(return_message,list): return_message[:] = [msg] + if msg.int_info < 0: + try: + err_msg = iRODSMessage(msg=msg.error).get_main_message(Error).RErrMsg_PI[0].msg + except TypeError: + err_msg = None + if nominal_code(msg.int_info) not in acceptable_codes: + raise get_exception_by_code(msg.int_info, err_msg) + return msg + + def recv_into(self, buffer, **options): + return self.recv( into_buffer = buffer, **options ) + + def reply(self, api_reply_index): + value = socket.htonl(api_reply_index) + try: + self.socket.sendall(struct.pack('I', value)) + except: + #self.release(True) + raise NetworkException("Unable to send API reply") + + def requires_cs_negotiation(self): + try: + if self.account.client_server_negotiation == REQUEST_NEGOTIATION: + return True + except AttributeError: + return False + return False + + @staticmethod + def make_ssl_context(irods_account): + check_hostname = getattr(irods_account,'ssl_verify_server','hostname') + CAfile = getattr(irods_account,'ssl_ca_certificate_file',None) + CApath = getattr(irods_account,'ssl_ca_certificate_path',None) + verify = ssl.CERT_NONE if (None is CAfile is CApath) else ssl.CERT_REQUIRED + # See https://stackoverflow.com/questions/30461969/disable-default-certificate-verification-in-python-2-7-9/49040695#49040695 + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=CAfile, capath=CApath) + # Note: check_hostname must be assigned prior to verify_mode property or Python library complains! + ctx.check_hostname = (check_hostname.startswith('host') and verify != ssl.CERT_NONE) + ctx.verify_mode = verify + return ctx + + def ssl_startup(self): + # Get encryption settings from client environment + host = self.account.host + algo = self.account.encryption_algorithm + key_size = self.account.encryption_key_size + hash_rounds = self.account.encryption_num_hash_rounds + salt_size = self.account.encryption_salt_size + + try: + context = self.account.ssl_context + except AttributeError: + self.account.ssl_context = context = self.make_ssl_context(self.account) + + # Wrap socket with context + wrapped_socket = context.wrap_socket(self.socket, + server_hostname=(host if context.check_hostname else None)) + + # Initial SSL handshake + wrapped_socket.do_handshake() + + # Generate key (shared secret) + key = os.urandom(self.account.encryption_key_size) + + # Send header-only message with client side encryption settings + packed_header = iRODSMessage.pack_header(algo, + key_size, + salt_size, + hash_rounds, + 0) + wrapped_socket.sendall(packed_header) + + # Send shared secret + packed_header = iRODSMessage.pack_header('SHARED_SECRET', + key_size, + 0, + 0, + 0) + wrapped_socket.sendall(packed_header + key) + + # Use SSL socket from now on + self.socket = wrapped_socket + + def _connect(self): + address = (self.account.host, self.account.port) + timeout = None + #timeout = self.pool.connection_timeout # dwm - PRC mk 2 + + try: + s = socket.create_connection(address, timeout) + self._disconnected = False + except socket.error: + raise NetworkException( + "Could not connect to specified host and port: " + + "{}:{}".format(*address)) + + self.socket = s + + main_message = StartupPack( + (self.account.proxy_user, self.account.proxy_zone), + (self.account.client_user, self.account.client_zone), + self.application_name + ) + + # No client-server negotiation + if not self.requires_cs_negotiation(): + + # Send startup pack without negotiation request + msg = iRODSMessage(msg_type='RODS_CONNECT', msg=main_message) + self.send(msg) + + # Server responds with version + version_msg = self.recv() + + # Done + return version_msg.get_main_message(VersionResponse) + + # Get client negotiation policy + client_policy = getattr(self.account, 'client_server_policy', REQUIRE_TCP) + + # Sanity check + validate_policy(client_policy) + + # Send startup pack with negotiation request + main_message.option = '{};{}'.format(main_message.option, REQUEST_NEGOTIATION) + msg = iRODSMessage(msg_type='RODS_CONNECT', msg=main_message) + self.send(msg) + + # Server responds with its own negotiation policy + cs_neg_msg = self.recv() + response = cs_neg_msg.get_main_message(ClientServerNegotiation) + server_policy = response.result + + # Perform the negotiation + neg_result, status = perform_negotiation(client_policy=client_policy, + server_policy=server_policy) + + # Send negotiation result to server + client_neg_response = ClientServerNegotiation( + status=status, + result='{}={};'.format(CS_NEG_RESULT_KW, neg_result) + ) + msg = iRODSMessage(msg_type='RODS_CS_NEG_T', msg=client_neg_response) + self.send(msg) + + # If negotiation failed we're done + if neg_result == FAILURE: + self.disconnect() + raise NetworkException("Client-Server negotiation failure: {},{}".format(client_policy, server_policy)) + + # Server responds with version + version_msg = self.recv() + + if neg_result == USE_SSL: + self.ssl_startup() + + return version_msg.get_main_message(VersionResponse) + + def disconnect(self): + # Moved the conditions to call disconnect() inside the function. + # Added a new criteria for calling disconnect(); Only call + # disconnect() if fileno is not -1 (fileno -1 indicates the socket + # is already closed). This makes it safe to call disconnect multiple + # times on the same connection. The first call cleans up the resources + # and next calls are no-ops + try: + if self.socket and getattr(self, "_disconnected", False) == False and self.socket.fileno() != -1: + disconnect_msg = iRODSMessage(msg_type='RODS_DISCONNECT') + self.send(disconnect_msg) + try: + # SSL shutdown handshake + self.socket = self.socket.unwrap() + except AttributeError: + pass + self.socket.shutdown(socket.SHUT_RDWR) + self.socket.close() + finally: + self._disconnected = True # Issue 368 - because of undefined destruction order during interpreter shutdown, + self.socket = None # as well as the fact that unhandled exceptions are ignored in __del__, we'd at least + # like to ensure as much cleanup as possible, thus preventing the above socket shutdown + # procedure from running too many times and creating confusing messages + + def recvall(self, n): + # Helper function to recv n bytes or return None if EOF is hit + data = b'' + while len(data) < n: + packet = self.socket.recv(n - len(data)) + if not packet: + return None + data += packet + return data + + def init_sec_context(self): + import gssapi + + # AUTHORIZATION MECHANISM + gsi_mech = gssapi.raw.OID.from_int_seq(GSI_OID) + + # SERVER NAME + server_name = gssapi.Name(self.account.server_dn) + server_name.canonicalize(gsi_mech) + + # CLIENT CONTEXT + self.client_ctx = gssapi.SecurityContext( + name=server_name, + mech=gsi_mech, + flags=[2, 4], + usage='initiate') + + def send_gsi_token(self, server_token=None): + + # CLIENT TOKEN + if server_token is None: + client_token = self.client_ctx.step() + else: + client_token = self.client_ctx.step(server_token) + logger.debug("[GSI handshake] Client: sent a new token") + + # SEND IT TO SERVER + self.reply(len(client_token)) + self.socket.sendall(client_token) + + def receive_gsi_token(self): + + # Receive client token from iRODS + data = self.socket.recv(4) + value = struct.unpack("I", bytearray(data)) + token_len = socket.ntohl(value[0]) + server_token = self.recvall(token_len) + logger.debug("[GSI handshake] Server: received a new token") + + return server_token + + def handshake(self, target): + """ + This GSS API context based on GSI was obtained combining 2 sources: + https://pythonhosted.org/gssapi/basic-tutorial.html + https://github.com/irods/irods_auth_plugin_gsi/blob/master/gsi/libgsi.cpp + """ + + self.init_sec_context() + + # Go, into the loop + self.send_gsi_token() + + while not (self.client_ctx.complete): + + server_token = self.receive_gsi_token() + + self.send_gsi_token(server_token) + + logger.debug("[GSI Handshake] completed") + + def gsi_client_auth_request(self): + + # Request for authentication with GSI on current user + + message_body = PluginAuthMessage( + auth_scheme_=GSI_AUTH_PLUGIN, + context_='%s=%s' % (AUTH_USER_KEY, self.account.client_user) + ) + # GSI = 1201 +# https://github.com/irods/irods/blob/master/lib/api/include/apiNumber.h#L158 + auth_req = iRODSMessage( + msg_type='RODS_API_REQ', msg=message_body, int_info=1201) + self.send(auth_req) + # Getting the challenge message + self.recv() + + # This receive an empty message for confirmation... To check: + # challenge_msg = self.recv() + + def gsi_client_auth_response(self): + + message = '%s=%s' % (AUTH_SCHEME_KEY, GSI_AUTH_SCHEME) + # IMPORTANT! padding + len_diff = RESPONSE_LEN - len(message) + message += "\0" * len_diff + + # mimic gsi_auth_client_response + gsi_msg = AuthResponse( + response=message, + username=self.account.proxy_user + '#' + self.account.proxy_zone + ) + gsi_request = iRODSMessage( + msg_type='RODS_API_REQ', int_info=api_number['AUTH_RESPONSE_AN'], msg=gsi_msg) + self.send(gsi_request) + self.recv() + # auth_response = self.recv() + + def _login_gsi(self): + # Send iRODS server a message to request GSI authentication + self.gsi_client_auth_request() + + # Create a context handshaking GSI credentials + # Note: this can work only if you export GSI certificates + # as shell environment variables (X509_etc.) + self.handshake(self.account.host) + + # Complete the protocol + self.gsi_client_auth_response() + + logger.info("GSI authorization validated") + + def _login_pam(self): + + time_to_live_in_seconds = 60 + + pam_password = PAM_PW_ESC_PATTERN.sub(lambda m: '\\'+m.group(1), self.account.password) + + ctx_user = '%s=%s' % (AUTH_USER_KEY, self.account.client_user) + ctx_pwd = '%s=%s' % (AUTH_PWD_KEY, pam_password) + ctx_ttl = '%s=%s' % (AUTH_TTL_KEY, str(time_to_live_in_seconds)) + + ctx = ";".join([ctx_user, ctx_pwd, ctx_ttl]) + + if type(self.socket) is socket.socket: + if getattr(self,'DISALLOWING_PAM_PLAINTEXT',True): + raise PlainTextPAMPasswordError + + Pam_Long_Tokens = (ALLOW_PAM_LONG_TOKENS and (len(ctx) >= MAX_NAME_LEN)) + + if Pam_Long_Tokens: + + message_body = PamAuthRequest( + pamUser=self.account.client_user, + pamPassword=pam_password, + timeToLive=time_to_live_in_seconds) + else: + + message_body = PluginAuthMessage( + auth_scheme_ = PAM_AUTH_SCHEME, + context_ = ctx) + + auth_req = iRODSMessage( + msg_type='RODS_API_REQ', + msg=message_body, + int_info=(725 if Pam_Long_Tokens else 1201) + ) + + self.send(auth_req) + # Getting the new password + output_message = self.recv() + + Pam_Response_Class = (PamAuthRequestOut if Pam_Long_Tokens + else AuthPluginOut) + + auth_out = output_message.get_main_message( Pam_Response_Class ) + + self.disconnect() + self._connect() + + if hasattr(self.account,'store_pw'): + drop = self.account.store_pw + if type(drop) is list: + drop[:] = [ auth_out.result_ ] + + self._login_native(password=auth_out.result_) + + logger.info("PAM authorization validated") + + def read_file(self, desc, size=-1, buffer=None): + if size < 0: + size = len(buffer) + elif buffer is not None: + size = min(size, len(buffer)) + + message_body = OpenedDataObjRequest( + l1descInx=desc, + len=size, + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap() + ) + message = iRODSMessage('RODS_API_REQ', msg=message_body, + int_info=api_number['DATA_OBJ_READ_AN']) + + logger.debug(desc) + self.send(message) + if buffer is None: + response = self.recv() + else: + response = self.recv_into(buffer) + + return response.bs + + def _login_native(self, password=None): + + # Default case, PAM login will send a new password + if password is None: + password = self.account.password or '' + + # authenticate + auth_req = iRODSMessage(msg_type='RODS_API_REQ', int_info=703) + self.send(auth_req) + + # challenge + challenge_msg = self.recv() + logger.debug(challenge_msg.msg) + challenge = challenge_msg.get_main_message(AuthChallenge).challenge + + # one "session" signature per connection + # see https://github.com/irods/irods/blob/4.2.1/plugins/auth/native/libnative.cpp#L137 + # and https://github.com/irods/irods/blob/4.2.1/lib/core/src/clientLogin.cpp#L38-L60 + if six.PY2: + self._client_signature = "".join("{:02x}".format(ord(c)) for c in challenge[:16]) + else: + self._client_signature = "".join("{:02x}".format(c) for c in challenge[:16]) + + if six.PY3: + challenge = challenge.strip() + padded_pwd = struct.pack( + "%ds" % MAX_PASSWORD_LENGTH, password.encode( + 'utf-8').strip()) + else: + padded_pwd = struct.pack( + "%ds" % MAX_PASSWORD_LENGTH, password) + + m = hashlib.md5() + m.update(challenge) + m.update(padded_pwd) + encoded_pwd = m.digest() + + if six.PY2: + encoded_pwd = encoded_pwd.replace('\x00', '\x01') + elif b'\x00' in encoded_pwd: + encoded_pwd_array = bytearray(encoded_pwd) + encoded_pwd = bytes(encoded_pwd_array.replace(b'\x00', b'\x01')) + + + pwd_msg = AuthResponse( + response=encoded_pwd, username=self.account.proxy_user) + pwd_request = iRODSMessage( + msg_type='RODS_API_REQ', int_info=api_number['AUTH_RESPONSE_AN'], msg=pwd_msg) + self.send(pwd_request) + self.recv() + + def write_file(self, desc, string): + message_body = OpenedDataObjRequest( + l1descInx=desc, + len=len(string), + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap() + ) + message = iRODSMessage('RODS_API_REQ', msg=message_body, + bs=string, + int_info=api_number['DATA_OBJ_WRITE_AN']) + self.send(message) + response = self.recv() + return response.int_info + + def seek_file(self, desc, offset, whence): + message_body = OpenedDataObjRequest( + l1descInx=desc, + len=0, + whence=whence, + oprType=0, + offset=offset, + bytesWritten=0, + KeyValPair_PI=StringStringMap() + ) + message = iRODSMessage('RODS_API_REQ', msg=message_body, + int_info=api_number['DATA_OBJ_LSEEK_AN']) + + self.send(message) + response = self.recv() + offset = response.get_main_message(FileSeekResponse).offset + return offset + + def close_file(self, desc, **options): + message_body = OpenedDataObjRequest( + l1descInx=desc, + len=0, + whence=0, + oprType=0, + offset=0, + bytesWritten=0, + KeyValPair_PI=StringStringMap(options) + ) + message = iRODSMessage('RODS_API_REQ', msg=message_body, + int_info=api_number['DATA_OBJ_CLOSE_AN']) + + self.send(message) + self.recv() + + def temp_password(self): + request = iRODSMessage("RODS_API_REQ", msg=None, + int_info=api_number['GET_TEMP_PASSWORD_AN']) + + # Send and receive request + self.send(request) + response = self.recv() + logger.debug(response.int_info) + + # Convert and return answer + msg = response.get_main_message(GetTempPasswordOut) + return obf.create_temp_password(msg.stringToHashWith, self.account.password) From f0ff6939054214f175fd7bd961d92796bd6ed38b Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Wed, 12 Jul 2023 17:44:22 -0400 Subject: [PATCH 2/7] alphabetize imports, require python >= 3.6 --- irods/simple_client/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/irods/simple_client/__init__.py b/irods/simple_client/__init__.py index 34e700ff..86c05f6a 100644 --- a/irods/simple_client/__init__.py +++ b/irods/simple_client/__init__.py @@ -1,5 +1,7 @@ -from __future__ import print_function -from __future__ import absolute_import +#from __future__ import absolute_import +import sys +if sys.version_info < (3, 6): + raise RuntimeError('Python3.6 is required') import atexit import ast @@ -11,14 +13,14 @@ import json import logging import os -import socket -import struct +import re import six -import os +import socket import ssl +import struct + import irods.password_obfuscation as obf from irods import MAX_NAME_LEN -import re from irods.password_obfuscation import decode from irods.account import iRODSAccount @@ -37,7 +39,7 @@ def __enter__(self): self.pool.get_connection() return self - def __exit__(self,*x): + def __exit__(self,*_): self.cleanup() return From 37bae4a879b637c7bc87dfcec1df460ecfa2369b Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Wed, 12 Jul 2023 18:20:47 -0400 Subject: [PATCH 3/7] limit how many object ref-cycles can accumulate in GC --- irods/manager/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/irods/manager/__init__.py b/irods/manager/__init__.py index 09c184c2..5d9d1a39 100644 --- a/irods/manager/__init__.py +++ b/irods/manager/__init__.py @@ -13,3 +13,10 @@ def server_version(self): def __init__(self, sess): self.sess = sess + +def _setup_for_limiting_connections(): + import gc + # When sessions and managers go out of scope, clean them up with some rapidity: + gc.set_threshold(32) + +_setup_for_limiting_connections() From 0e25b52105158c7d0d83aa056ce21df5fa48c8af Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Wed, 12 Jul 2023 18:18:24 -0400 Subject: [PATCH 4/7] show we still can use traditional managers --- irods/simple_client/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/irods/simple_client/__init__.py b/irods/simple_client/__init__.py index 86c05f6a..a80f047f 100644 --- a/irods/simple_client/__init__.py +++ b/irods/simple_client/__init__.py @@ -24,6 +24,8 @@ from irods.password_obfuscation import decode from irods.account import iRODSAccount +from irods.manager.data_object_manager import DataObjectManager +from irods.manager.collection_manager import CollectionManager from irods import NATIVE_AUTH_SCHEME, PAM_AUTH_SCHEME from irods.query import Query @@ -77,6 +79,8 @@ def __init__(self, configure = True, **kwargs): self._auth_file = '' self.account = None self.conn = None + self.data_objects = DataObjectManager(self) + self.collections = CollectionManager(self) if configure: self.account = self.configure(**kwargs) From e3b4313c88693b5c9190ad522fa74de27b52cdf4 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Fri, 14 Jul 2023 08:23:39 -0400 Subject: [PATCH 5/7] ticket,clone host --- irods/simple_client/__init__.py | 16 +++++++++++++--- irods/test_client/applyticket.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100755 irods/test_client/applyticket.py diff --git a/irods/simple_client/__init__.py b/irods/simple_client/__init__.py index a80f047f..ef787ae7 100644 --- a/irods/simple_client/__init__.py +++ b/irods/simple_client/__init__.py @@ -28,6 +28,7 @@ from irods.manager.collection_manager import CollectionManager from irods import NATIVE_AUTH_SCHEME, PAM_AUTH_SCHEME from irods.query import Query +from irods.ticket import Ticket logger = logging.getLogger(__name__) @@ -45,9 +46,12 @@ def __exit__(self,*_): self.cleanup() return - def clone(self): + def clone(self, *, + host = ''): other = copy.copy(self) other.account = copy.copy(self.account) + if host: + other.account.host = host other.conn = None return other @@ -71,14 +75,20 @@ class _: def get_connection(): if self.conn is None: self.conn = Connection(self.account) + if self.ticket__: + Ticket(self, self.ticket__).supply() return self.conn - return _() + return _() # -> allows source level compatibility with much of existing PRC as of version 1.1.8 + # (many low-level calls throughout PRC use session.pool.get_connection()) - def __init__(self, configure = True, **kwargs): + def __init__(self, *, configure = True, + ticket = '', + **kwargs): self._env_file = '' self._auth_file = '' self.account = None self.conn = None + self.ticket__ = ticket self.data_objects = DataObjectManager(self) self.collections = CollectionManager(self) if configure: diff --git a/irods/test_client/applyticket.py b/irods/test_client/applyticket.py new file mode 100755 index 00000000..8060a05e --- /dev/null +++ b/irods/test_client/applyticket.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +import sys,os +import irods.test.helpers as h +from irods.simple_client import Session +from irods.ticket import Ticket +#s = h.make_session() + +ENV_FILE = irods_env_file=os.path.expanduser('~/.irods/irods_environment.json') +s = Session(irods_env_file = ENV_FILE) + +if len(sys.argv) > 1: + Ticket(s,sys.argv[1]).supply() +from irods.models import DataObject,Collection +q = s.query( DataObject.name,Collection.name) + +print('data objects:') +print('-------------') +for d in list(q): + print( d[Collection.name],'/', + d[DataObject.name],sep='') From 34ceb20b4fe6dfbecb36f3d6f4e8bc4587b4c3e7 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Sun, 16 Jul 2023 12:03:10 -0400 Subject: [PATCH 6/7] restructure, make requests a dependency --- irods/client/http/__init__.py | 7 +++++++ irods/{simple_client => client/simple}/__init__.py | 0 setup.py | 1 + 3 files changed, 8 insertions(+) create mode 100644 irods/client/http/__init__.py rename irods/{simple_client => client/simple}/__init__.py (100%) diff --git a/irods/client/http/__init__.py b/irods/client/http/__init__.py new file mode 100644 index 00000000..cb5415a4 --- /dev/null +++ b/irods/client/http/__init__.py @@ -0,0 +1,7 @@ +import requests + +class Session: + def __init__(username, password): + self.username = username + self.password = password + diff --git a/irods/simple_client/__init__.py b/irods/client/simple/__init__.py similarity index 100% rename from irods/simple_client/__init__.py rename to irods/client/simple/__init__.py diff --git a/setup.py b/setup.py index d280ced1..bb3ee882 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'six>=1.10.0', 'PrettyTable>=0.7.2', 'defusedxml', + 'requests', # - the new syntax: #'futures; python_version == "2.7"' ], From e24ba9901ff033026abfe2424c828ae06f7cefc0 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Sun, 16 Jul 2023 18:37:34 -0400 Subject: [PATCH 7/7] http client --- irods/client/http/__init__.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/irods/client/http/__init__.py b/irods/client/http/__init__.py index cb5415a4..902a4576 100644 --- a/irods/client/http/__init__.py +++ b/irods/client/http/__init__.py @@ -1,7 +1,32 @@ import requests +import collections + +class HTTP_connect_failed(RuntimeError): + pass class Session: - def __init__(username, password): + + url_base_template = 'http://{self.host}:{self.port}/irods-http-api/{self.version}' + + @property + def url_base(self): + return self.url_base_template.format(**locals()) + + def __init__(self, username, password, *, + host = 'localhost', + port = 9000, + version = '0.9.5'): + self.username = username self.password = password + (self.host, self.port, self.version) = (host, port, version) + + r = requests.post(self.url_base + '/authenticate', auth=(self.username, self.password)) + if r.status_code != 200 or not r.text: + raise HTTP_connect_failed + self.bearer_token = r.text + + +if __name__ == '__main__': + Session('rods','rods',host='192.168.0.5')