from NGPIris.parse_credentials import CredentialsHandler
from NGPIris.hcp.helpers import (
raise_path_error,
create_access_control_policy,
check_mounted
)
from NGPIris.hcp.exceptions import *
from boto3 import client
from botocore.client import Config
from botocore.exceptions import EndpointConnectionError, ClientError
from boto3.s3.transfer import TransferConfig
from configparser import ConfigParser
from os import (
path,
stat,
listdir
)
from json import dumps
from parse import (
parse,
search,
Result
)
from requests import get
from urllib3 import disable_warnings
from tqdm import tqdm
_KB = 1024
_MB = _KB * _KB
[docs]
class HCPHandler:
def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path : str = "", custom_config_path : str = "") -> None:
"""
Class for handling HCP requests.
:param credentials_path: Path to the JSON credentials file
:type credentials_path: str
:param use_ssl: Boolean choice between using SSL, defaults to False
:type use_ssl: bool, optional
:param custom_config_path: Path to a .ini file for customs settings regarding download and upload
:type custom_config_path: str, optional
"""
credentials_handler = CredentialsHandler(credentials_path)
self.hcp = credentials_handler.hcp
self.endpoint = "https://" + self.hcp["endpoint"]
tenant_parse = parse("https://{}.hcp1.vgregion.se", self.endpoint)
if type(tenant_parse) is Result:
self.tenant = str(tenant_parse[0])
else: # pragma: no cover
raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers")
self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant
self.aws_access_key_id = self.hcp["aws_access_key_id"]
self.aws_secret_access_key = self.hcp["aws_secret_access_key"]
self.token = self.aws_access_key_id + ":" + self.aws_secret_access_key
self.bucket_name = None
self.use_ssl = use_ssl
if not self.use_ssl:
disable_warnings()
if proxy_path: # pragma: no cover
s3_config = Config(
s3 = {
"addressing_style": "path",
"payload_signing_enabled": True
},
signature_version = "s3v4",
proxies = CredentialsHandler(proxy_path).hcp
)
else:
s3_config = Config(
s3 = {
"addressing_style": "path",
"payload_signing_enabled": True
},
signature_version = "s3v4"
)
self.s3_client = client(
"s3",
aws_access_key_id = self.aws_access_key_id,
aws_secret_access_key = self.aws_secret_access_key,
endpoint_url = self.endpoint,
verify = self.use_ssl,
config = s3_config
)
if custom_config_path: # pragma: no cover
ini_config = ConfigParser()
ini_config.read(custom_config_path)
self.transfer_config = TransferConfig(
multipart_threshold = ini_config.getint("hcp", "multipart_threshold"),
max_concurrency = ini_config.getint("hcp", "max_concurrency"),
multipart_chunksize = ini_config.getint("hcp", "multipart_chunksize"),
use_threads = ini_config.getboolean("hcp", "use_threads")
)
else:
self.transfer_config = TransferConfig(
multipart_threshold = 10 * _MB,
max_concurrency = 60,
multipart_chunksize = 40 * _MB,
use_threads = True
)
[docs]
def get_response(self, path_extension : str = "") -> dict:
"""
Make a request to the HCP in order to use the builtin MAPI
:param path_extension: Extension for the base request URL, defaults to the empty string
:type path_extension: str, optional
:return: The response as a dictionary
:rtype: dict
"""
url = self.base_request_url + path_extension
headers = {
"Authorization": "HCP " + self.token,
"Cookie": "hcp-ns-auth=" + self.token,
"Accept": "application/json"
}
response = get(
url,
headers=headers,
verify=self.use_ssl
)
response.raise_for_status()
return dict(response.json())
[docs]
def test_connection(self, bucket_name : str = "") -> dict:
"""
Test the connection to the mounted bucket or another bucket which is
supplied as the argument :py:obj:`bucket_name`.
:param bucket_name: The name of the bucket to be mounted. Defaults to the empty string
:type bucket_name: str, optional
:raises RuntimeError: If no bucket is selected
:raises VPNConnectionError: If there is no VPN connection
:raises BucketNotFound: If no bucket of that name was found
:raises Exception: Other exceptions
:return: A dictionary of the response
:rtype: dict
"""
if not bucket_name and self.bucket_name:
bucket_name = self.bucket_name
elif bucket_name:
pass
else:
raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`")
try:
response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
except EndpointConnectionError as e: # pragma: no cover
print(e)
raise VPNConnectionError("Please check your connection and that you have your VPN enabled")
except ClientError as e:
print(e)
raise BucketNotFound("Bucket \"" + bucket_name + "\" was not found")
except Exception as e: # pragma: no cover
raise Exception(e)
if response["ResponseMetadata"].get("HTTPStatusCode", -1) != 200: # pragma: no cover
error_msg = "The response code from the request made at " + self.endpoint + " returned status code " + response["ResponseMetadata"]["HTTPStatusCode"]
raise Exception(error_msg)
return response
[docs]
def mount_bucket(self, bucket_name : str) -> None:
"""
Mount bucket that is to be used. This method needs to executed in order
for most of the other methods to work. It mainly concerns operations with
download and upload.
:param bucket_name: The name of the bucket to be mounted
:type bucket_name: str
"""
# Check if bucket exist
self.test_connection(bucket_name = bucket_name)
self.bucket_name = bucket_name
[docs]
def list_buckets(self) -> list[str]:
"""
List all available buckets at endpoint.
:return: A list of buckets
:rtype: list[str]
"""
response = self.get_response("/namespaces")
list_of_buckets : list[str] = response["name"]
return list_of_buckets
@check_mounted
def list_objects(self, name_only : bool = False) -> list:
"""
List all objects in the mounted bucket
:param name_only: If True, return only a list of the object names. If False, return the full metadata about each object. Defaults to False.
:type name_only: bool, optional
:return: A list of of either strings or a list of object metadata (the form of a dictionary)
:rtype: list
"""
response_list_objects = dict(self.s3_client.list_objects_v2(
Bucket = self.bucket_name
))
if "Contents" not in response_list_objects.keys(): # pragma: no cover
return []
list_of_objects : list[dict] = response_list_objects["Contents"]
if name_only:
return [object["Key"] for object in list_of_objects]
else:
return list_of_objects
@check_mounted
def get_object(self, key : str) -> dict:
"""
Retrieve object metadata
:param key: The object name
:type key: str
:return: A dictionary containing the object metadata
:rtype: dict
"""
response = dict(self.s3_client.get_object(
Bucket = self.bucket_name,
Key = key
))
return response
@check_mounted
def object_exists(self, key : str) -> bool:
"""
Check if a given object is in the mounted bucket
:param key: The object name
:type key: str
:return: True if the object exist, otherwise False
:rtype: bool
"""
try:
response = self.get_object(key)
if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
return True
else: # pragma: no cover
return False
except: # pragma: no cover
return False
@check_mounted
def download_file(self, key : str, local_file_path : str) -> None:
"""
Download one object file from the mounted bucket
:param key: Name of the object
:type key: str
:param local_file_path: Path to a file on your local system where the contents of the object file can be put.
:type local_file_path: str
"""
try:
file_size : int = self.s3_client.head_object(Bucket = self.bucket_name, Key = key)["ContentLength"]
with tqdm(
total = file_size,
unit = "B",
unit_scale = True,
desc = key
) as pbar:
self.s3_client.download_file(
Bucket = self.bucket_name,
Key = key,
Filename = local_file_path,
Config = self.transfer_config,
Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
)
except ClientError as e0:
print(str(e0))
raise Exception("Could not find object", "\"" + key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
except Exception as e: # pragma: no cover
raise Exception(e)
@check_mounted
def upload_file(self, local_file_path : str, key : str = "") -> None:
"""
Upload one file to the mounted bucket
:param local_file_path: Path to the file to be uploaded
:type local_file_path: str
:param key: An optional new name for the file object on the bucket. Defaults to the same name as the file
:type key: str, optional
"""
raise_path_error(local_file_path)
if not key:
file_name = path.basename(local_file_path)
key = file_name
file_size : int = stat(local_file_path).st_size
with tqdm(
total = file_size,
unit = "B",
unit_scale = True,
desc = local_file_path
) as pbar:
self.s3_client.upload_file(
Filename = local_file_path,
Bucket = self.bucket_name,
Key = key,
Config = self.transfer_config,
Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
)
@check_mounted
def upload_folder(self, local_folder_path : str, key : str = "") -> None:
"""
Upload the contents of a folder to the mounted bucket
:param local_folder_path: Path to the folder to be uploaded
:type local_folder_path: str
:param key: An optional new name for the folder path on the bucket. Defaults to the same name as the local folder path
:type key: str, optional
"""
raise_path_error(local_folder_path)
if not key:
key = local_folder_path
filenames = listdir(local_folder_path)
for filename in filenames:
self.upload_file(local_folder_path + filename, key + filename)
@check_mounted
def delete_objects(self, keys : list[str], verbose : bool = True) -> None:
"""Delete a list of objects on the mounted bucket
:param keys: List of object names to be deleted
:type keys: list[str]
:param verbose: Print the result of the deletion. Defaults to True
:type verbose: bool, optional
"""
object_list = []
for key in keys:
object_list.append({"Key" : key})
deletion_dict = {"Objects": object_list}
list_of_objects_before = self.list_objects(True)
response : dict = self.s3_client.delete_objects(
Bucket = self.bucket_name,
Delete = deletion_dict
)
if verbose:
print(dumps(response, indent=4))
diff : set[str] = set(keys) - set(list_of_objects_before)
if diff:
does_not_exist = []
for key in diff:
does_not_exist.append("- " + key + "\n")
print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist))
@check_mounted
def delete_object(self, key : str, verbose : bool = True) -> None:
"""
Delete a single object in the mounted bucket
:param key: The object to be deleted
:type key: str
:param verbose: Print the result of the deletion. Defaults to True
:type verbose: bool, optional
"""
self.delete_objects([key], verbose = verbose)
@check_mounted
def delete_folder(self, key : str, verbose : bool = True) -> None:
"""
Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raisesd
:param key: The folder of objects to be deleted
:type key: str
:param verbose: Print the result of the deletion. defaults to True
:type verbose: bool, optional
:raises RuntimeError: If there are subfolders, a RuntimeError is raisesd
"""
if key[-1] != "/":
key += "/"
object_path_in_folder = []
for s in self.search_objects_in_bucket(key):
parse_object = parse(key + "{}", s)
if type(parse_object) is Result:
object_path_in_folder.append(s)
for object_path in object_path_in_folder:
if object_path[-1] == "/":
raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one")
self.delete_objects(object_path_in_folder + [key], verbose = verbose)
@check_mounted
def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]:
"""
Simple search method using substrings in order to find certain objects. Case insensitive by default.
:param search_string: Substring to be used in the search
:type search_string: str
:param case_sensitive: Case sensitivity. Defaults to False
:type case_sensitive: bool, optional
:return: List of object names that match the in some way to the object names
:rtype: list[str]
"""
search_result : list[str] = []
for key in self.list_objects(True):
parse_object = search(
search_string,
key,
case_sensitive = case_sensitive
)
if type(parse_object) is Result:
search_result.append(key)
return search_result
@check_mounted
def get_object_acl(self, key : str) -> dict:
"""
Get the object Access Control List (ACL)
:param key: The name of the object
:type key: str
:return: Return the ACL in the shape of a dictionary
:rtype: dict
"""
response : dict = self.s3_client.get_object_acl(
Bucket = self.bucket_name,
Key = key
)
return response
@check_mounted
def get_bucket_acl(self) -> dict:
"""
Get the bucket Access Control List (ACL)
:return: Return the ACL in the shape of a dictionary
:rtype: dict
"""
response : dict = self.s3_client.get_bucket_acl(
Bucket = self.bucket_name
)
return response
@check_mounted
def modify_single_object_acl(self, key : str, user_ID : str, permission : str) -> None:
"""
Modify permissions for a user in the Access Control List (ACL) for one object
:param key: The name of the object
:type key: str
:param user_ID: The user name. Can either be the DisplayName or user_ID
:type user_ID: str
:param permission:
What permission to be set. Valid options are:
* FULL_CONTROL
* WRITE
* WRITE_ACP
* READ
* READ_ACP\n
:type permission: str
"""
self.s3_client.put_object_acl(
Bucket = self.bucket_name,
Key = key,
AccessControlPolicy = create_access_control_policy({user_ID : permission})
)
@check_mounted
def modify_single_bucket_acl(self, user_ID : str, permission : str) -> None:
"""
Modify permissions for a user in the Access Control List (ACL) for the mounted bucket
:param user_ID: The user name. Can either be the DisplayName or user_ID
:type user_ID: str
:param permission:
What permission to be set. Valid options are:
* FULL_CONTROL
* WRITE
* WRITE_ACP
* READ
* READ_ACP\n
:type permission: str
"""
self.s3_client.put_bucket_acl(
Bucket = self.bucket_name,
AccessControlPolicy = create_access_control_policy({user_ID : permission})
)
@check_mounted
def modify_object_acl(self, key_user_ID_permissions : dict[str, dict[str, str]]) -> None:
"""
Modifies permissions to multiple objects, see below.
In order to add permissions for multiple objects, we make use of a dictionary of a dictionary: :py:obj:`key_user_ID_permissions = {key : {user_ID : permission}}`. So for every object (key), we set the permissions for every user ID for that object.
:param key_user_ID_permissions: The dictionary containing object name and user_id-permission dictionary
:type key_user_ID_permissions: dict[str, dict[str, str]]
"""
for key, user_ID_permissions in key_user_ID_permissions.items():
self.s3_client.put_object_acl(
Bucket = self.bucket_name,
Key = key,
AccessControlPolicy = create_access_control_policy(user_ID_permissions)
)
@check_mounted
def modify_bucket_acl(self, user_ID_permissions : dict[str, str]) -> None:
"""
Modify permissions for multiple users for the mounted bucket
:param user_ID_permissions: The dictionary containing the user name and the corresponding permission to be set to that user
:type user_ID_permissions: dict[str, str]
"""
self.s3_client.put_bucket_acl(
Bucket = self.bucket_name,
AccessControlPolicy = create_access_control_policy(user_ID_permissions)
)