diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index f04c93d..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Lint Code -on: [push, pull_request] -jobs: - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..d664bf9 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,31 @@ +name: Pre-commit Checks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install pre-commit if not already available in your dependencies + pip install pre-commit + # Optionally install your API package dependencies if needed + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Run pre-commit checks + run: pre-commit run diff --git a/.gitignore b/.gitignore index 90599c8..09bea1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ **/__pycache__/** *.bak -**/.ruff_cache/** \ No newline at end of file +**/.ruff_cache/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6ac71cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/psf/black + rev: 23.7.0 # Use the latest stable version available + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 # Update as appropriate + hooks: + - id: isort + language_version: python3 + + # - repo: https://github.com/PyCQA/flake8 + # rev: 6.1.0 # Update as needed + # hooks: + # - id: flake8 + # additional_dependencies: [flake8-bugbear] # optional extra checks + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 # Update as needed + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml diff --git a/README.md b/README.md index e6679de..77bf343 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,33 @@ # Bisque API for Python 3 [![Upload Python Package](https://github.com/UCSB-VRL/bqapi/actions/workflows/python-publish.yml/badge.svg)](https://github.com/UCSB-VRL/bqapi/actions/workflows/python-publish.yml) +[![Pre-commit Checks](https://github.com/UCSB-VRL/bqapi/actions/workflows/pre-commit.yml/badge.svg)](https://github.com/UCSB-VRL/bqapi/actions/workflows/pre-commit.yml) +[![PyPI](https://img.shields.io/pypi/v/bqapi-ucsb?style=flat-square)](https://pypi.org/project/bqapi-ucsb/) +[![Documentation](https://img.shields.io/badge/Documentation-View%20here-brightgreen?style=flat-square)](https://bisque.gitbook.io/docs) +[![PyPi Installs](https://pepy.tech/badge/bqapi-ucsb?style=flat-square)](https://pepy.tech/project/bqapi-ucsb) -[Full Documentation](https://github.com/UCSB-VRL/bisqueUCSB) -For development, follow [this guide](https://towardsdatascience.com/how-to-upload-your-python-package-to-pypi-de1b363a1b3) and [this repo](https://github.com/gmyrianthous/example-publish-pypi). -# Installing +## Install ``` pip install bqapi-ucsb ``` -[Pypi link](https://pypi.org/project/bqapi-ucsb) +## Usage -# Usage -```python -from bqapi.comm import BQSession -from bqapi.util import fetch_blob -``` +### Upload Image + +Upload your first image to BisQue using our tutorial! -# Development +[Upload an Image to BisQue](https://bisque.gitbook.io/docs/bisque-api/upload-an-image) + + +## Development For development, follow [this guide](https://towardsdatascience.com/how-to-upload-your-python-package-to-pypi-de1b363a1b3) and [this repo](https://github.com/gmyrianthous/example-publish-pypi). -## Added GitHub Action -### Summary +## Contribute 1. Clone repo 2. Make any necessary changes to source code, setup.py, and setup.cfg @@ -34,4 +36,3 @@ For development, follow [this guide](https://towardsdatascience.com/how-to-uploa 5. Make sure to have PyPi account credentials 6. run `twine upload dist/*` from main folder 7. Enter username and password - diff --git a/pyproject.toml b/pyproject.toml index 67f2b6f..53971a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,6 @@ # Follow Flake8's behavior lint.select = ["E", "F", "W", "C90"] # Enables Flake8-compatible rules lint.ignore = [] # Add rules to ignore -line-length = 79 -exclude = ["venv", ".git", "__pycache__", "**/tests/**"] -lint.mccabe.max-complexity = 25 # Set your desired threshold \ No newline at end of file +line-length = 79 +exclude = ["venv", ".git", "__pycache__", "**/tests/**"] +lint.mccabe.max-complexity = 25 # Set your desired threshold diff --git a/setup.py b/setup.py index 12fccd6..7926b9c 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,19 @@ -from setuptools import setup, find_packages - +from setuptools import find_packages, setup setup( - name='bqapi-ucsb', - version='1.0.6', + name="bqapi-ucsb", + version="1.0.6", author="Bisque Team", - author_email='amil@ucsb.edu', + author_email="amil@ucsb.edu", description="""Python API for interacting with BisQue""", - packages=find_packages('src'), - package_dir={'': 'src'}, - keywords='API Bisque', - url='https://github.com/UCSB-VRL/bisqueUCSB', + packages=find_packages("src"), + package_dir={"": "src"}, + keywords="API Bisque", + url="https://github.com/UCSB-VRL/bisqueUCSB", install_requires=[ - 'six', 'lxml', 'requests==2.32.3', 'requests-toolbelt', - ], + "six", + "lxml", + "requests==2.32.3", + "requests-toolbelt", + ], ) diff --git a/src/bqapi/RequestsMonkeyPatch/__init__.py b/src/bqapi/RequestsMonkeyPatch/__init__.py index ea30561..1bb8bf6 100644 --- a/src/bqapi/RequestsMonkeyPatch/__init__.py +++ b/src/bqapi/RequestsMonkeyPatch/__init__.py @@ -1 +1 @@ -#empty +# empty diff --git a/src/bqapi/RequestsMonkeyPatch/requests_patch.py b/src/bqapi/RequestsMonkeyPatch/requests_patch.py index 6da23f0..56bb01e 100644 --- a/src/bqapi/RequestsMonkeyPatch/requests_patch.py +++ b/src/bqapi/RequestsMonkeyPatch/requests_patch.py @@ -7,24 +7,25 @@ """ import email.utils - # import mimetypes import warnings import requests import requests.packages.urllib3 as urllib3 import six -from .monkeypatch import monkeypatch_method +from .monkeypatch import monkeypatch_method REQUESTS_V = [int(s) for s in requests.__version__.split(".")] print(REQUESTS_V) if REQUESTS_V < [2, 4, 0] or REQUESTS_V > [2, 32, 3]: - warnings.warn("""\ + warnings.warn( + """\ We need to patch requests 2.4.0 up to 2.32.3, make sure your version of requests needs this patch, greater than 2.4.3 we do not know if this -patch applys.""") +patch applys.""" + ) raise ImportError("Requests 2.4.0 to 2.32.3 is required!") # elif requests_v > [3, 0, 0]: # #does not require this patch @@ -46,7 +47,7 @@ def format_header_param(name, value): The value of the parameter, provided as a unicode string. """ if not any(ch in value for ch in '"\\\r\n'): - result = '%s="%s"' % (name, value) + result = '{}="{}"'.format(name, value) try: result.encode("ascii") except UnicodeEncodeError: @@ -58,7 +59,7 @@ def format_header_param(name, value): if not six.PY3: # Python 2: value_encode = value.encode("utf-8") - value = '%s="%s"; %s*=%s' % ( + value = '{}="{}"; {}*={}'.format( name, value, name, diff --git a/src/bqapi/blockable_module.py b/src/bqapi/blockable_module.py index 8c9f873..9246891 100644 --- a/src/bqapi/blockable_module.py +++ b/src/bqapi/blockable_module.py @@ -1,11 +1,8 @@ -import sys import collections - -from bqapi import BQSession - - import logging +import sys +from bqapi import BQSession # logging.basicConfig(filename='BlockableModule.log',level=logging.DEBUG) #!!! log = logging.getLogger("bqapi.blockable_module") @@ -21,17 +18,13 @@ def main(self, mex_url=None, auth_token=None, bq=None, **kw): # check for list parameters params = bq.get_mex_inputs() - if isinstance(params, dict) or not isinstance( - params, collections.Iterable - ): + if isinstance(params, dict) or not isinstance(params, collections.Iterable): params = [params] # pass values directly as args for single_params in params: for param_name in single_params: if "value" in single_params[param_name]: - single_params[param_name] = single_params[param_name].get( - "value" - ) + single_params[param_name] = single_params[param_name].get("value") # TODO: measure block startup time self.start_block(bq, params) diff --git a/src/bqapi/bqclass.py b/src/bqapi/bqclass.py index 45bfa7d..7c91469 100644 --- a/src/bqapi/bqclass.py +++ b/src/bqapi/bqclass.py @@ -51,25 +51,23 @@ __version__ = "0.1" __revision__ = "$Rev$" __date__ = "$Date$" -__copyright__ = ( - "Center for BioImage Informatics, University California, Santa Barbara" -) +__copyright__ = "Center for BioImage Informatics, University California, Santa Barbara" -import os -import sys -import math -import io +import collections import inspect +import io import logging +import math +import os +import sys import tempfile -import collections try: from lxml import etree except ImportError: import xml.etree.ElementTree as etree -from .xmldict import xml2nv +from .xmldict import xml2nv log = logging.getLogger("bqapi.class") @@ -148,9 +146,7 @@ def __repr__(self): def __str__(self): return "(%s:%s)" % ( self.xmltag, - ",".join( - ["%s=%s" % (f, getattr(self, f, "")) for f in self.xmlfields] - ), + ",".join(["%s=%s" % (f, getattr(self, f, "")) for f in self.xmlfields]), ) def toTuple(self): @@ -429,9 +425,7 @@ def fetch(self, path=None, stream=False): response = image_service.fetch(url, stream=stream) if path is not None: with open(path, "wb") as fb: - for block in response.iter_content( - chunk_size=16 * 1024 * 1024 - ): # 16MB + for block in response.iter_content(chunk_size=16 * 1024 * 1024): # 16MB fb.write(block) response.close() else: @@ -476,9 +470,7 @@ def asarray(self): # response.raw.decode_content = True return tifffile.imread(io.BytesIO(response.content)) - def savearray( - self, fname, imdata=None, imshape=None, dtype=None, **kwargs - ): + def savearray(self, fname, imdata=None, imshape=None, dtype=None, **kwargs): try: import tifffile except ImportError: diff --git a/src/bqapi/bqfeature.py b/src/bqapi/bqfeature.py index 937d0ed..87378f1 100644 --- a/src/bqapi/bqfeature.py +++ b/src/bqapi/bqfeature.py @@ -1,21 +1,22 @@ # import threading -from threading import Thread -import socket import errno -import tempfile -import urllib.request -import urllib.parse -import urllib.error +import logging import os -from math import ceil import queue -import logging +import socket +import tempfile +import urllib.error +import urllib.parse +import urllib.request import warnings from collections import namedtuple +from math import ceil +from threading import Thread + +from .exception import BQCommError # import numpy as np -from .exception import BQCommError try: # checks for lxml if not found uses python xml from lxml import etree @@ -36,11 +37,13 @@ FeatureResource = namedtuple("FeatureResource", ["image", "mask", "gobject"]) FeatureResource.__new__.__defaults__ = (None, None, None) + class FeatureError(Exception): """ Feature Communication Exception """ + class Feature(object): def fetch(self, session, name, resource_list, path=None): """ @@ -74,8 +77,7 @@ def fetch(self, session, name, resource_list, path=None): query = "&".join(query) sub.attrib["uri"] = "%s?%s" % (url, query) - log.debug("Fetch Feature %s for %s resources" % (name, - len(resource_list))) + log.debug("Fetch Feature %s for %s resources" % (name, len(resource_list))) if path is None: f = tempfile.NamedTemporaryFile( @@ -130,8 +132,7 @@ def fetch_vector(self, session, name, resource_list): status = status[index[0]][0] hdf5.close() os.remove(hdf5.filename) # remove file from temp directory - raise FeatureError("%s:Error occured during feature calculations" - % status) + raise FeatureError("%s:Error occured during feature calculations" % status) table = hdf5.root.values # status_table = hdf5.root.status # to get the status feature_vector = table[:]["feature"] @@ -150,8 +151,8 @@ def length(session, name): @return: feature length """ xml = session.fetchxml("/features/%s" % name) - return int(xml.find( - 'feature/tag[@name="feature_length"]').attrib.get("value")) + return int(xml.find('feature/tag[@name="feature_length"]').attrib.get("value")) + class ParallelFeature(Feature): MaxThread = 8 @@ -200,8 +201,7 @@ def run(self): else: break - def request_thread_pool(self, request_queue, errorcb=None, - thread_count=MaxThread): + def request_thread_pool(self, request_queue, errorcb=None, thread_count=MaxThread): """ Runs the BQRequestThread @@ -293,8 +293,7 @@ def fetch(self, session, name, resource_list, path=None): log.warning("Warning no resources were provided") return - log.debug("Exctracting %s on %s resources" % (name, - len(resource_list))) + log.debug("Exctracting %s on %s resources" % (name, len(resource_list))) if path is None: f = tempfile.TemporaryFile(suffix=".h5", dir=tempfile.gettempdir()) @@ -338,8 +337,7 @@ def run(self): temp_status_table = hdf5temp.root.status if not hasattr(hdf5.root, "values"): temp_table.copy(hdf5.root, "values") - temp_status_table.copy(hdf5.root, - "status") + temp_status_table.copy(hdf5.root, "status") else: table = hdf5.root.values status_table = hdf5.root.status @@ -420,8 +418,7 @@ def request(): else: thread_num, chunk_size = self.calculate_request_plan(resource_list) - for partial_resource_list in self.chunk(resource_list, - int(chunk_size)): + for partial_resource_list in self.chunk(resource_list, int(chunk_size)): request_queue.put(request_factory(partial_resource_list)) w = WriteHDF5Thread(write_queue) @@ -462,5 +459,4 @@ def fetch_vector(self, session, name, resource_list): @return: a list of features as numpy array """ - return super(ParallelFeature, self).fetch_vector(session, name, - resource_list) + return super(ParallelFeature, self).fetch_vector(session, name, resource_list) diff --git a/src/bqapi/casauth.py b/src/bqapi/casauth.py index aa930e0..3dcffb0 100644 --- a/src/bqapi/casauth.py +++ b/src/bqapi/casauth.py @@ -1,6 +1,6 @@ import logging -import requests +import requests from bs4 import BeautifulSoup as soupy diff --git a/src/bqapi/comm.py b/src/bqapi/comm.py index 972e7cb..fd9034e 100644 --- a/src/bqapi/comm.py +++ b/src/bqapi/comm.py @@ -51,21 +51,18 @@ """ -import sys - +import itertools # import urlparse # import urllib import logging -import itertools import posixpath - -from six.moves import urllib - +import sys import requests -from requests.auth import HTTPBasicAuth -from requests.auth import AuthBase from requests import Session +from requests.auth import AuthBase, HTTPBasicAuth +from six.moves import urllib + # from requests_toolbelt import MultipartEncoder try: @@ -79,11 +76,11 @@ except ImportError: import xml.etree.ElementTree as etree -from .bqclass import BQMex, BQNode, BQFactory +from .bqclass import BQFactory, BQMex, BQNode +from .exception import BQApiError, BQCommError +from .services import ServiceFactory # from .util import d2xml # parse_qs, make_qs, xml2d, d2xml, normalize_unicode from .xmldict import d2xml -from .services import ServiceFactory -from .exception import BQCommError, BQApiError try: from .casauth import caslogin @@ -116,7 +113,7 @@ def __init__(self, token, user=None): elif user in token.split(":")[0]: # check if token contains user self.username = "Mex %s" % (token) else: - self.username = "Mex %s:%s" % (user, token) + self.username = "Mex {}:{}".format(user, token) def __call__(self, r): """ @@ -135,7 +132,7 @@ class BQServer(Session): """ def __init__(self): - super(BQServer, self).__init__() + super().__init__() # Disable https session authentication.. # self.verify = False self.root = None @@ -196,9 +193,7 @@ def prepare_url(self, url, **params): raise BQApiError("No root provided") # query - query = [ - "%s=%s" % (k, v) for k, v in urllib.parse.parse_qsl(u.query, True) - ] + query = ["{}={}".format(k, v) for k, v in urllib.parse.parse_qsl(u.query, True)] unordered_query = [] ordered_query = [] @@ -210,16 +205,12 @@ def prepare_url(self, url, **params): ordered_query.append("%s=%s" % odict.popitem(False)) if params: - unordered_query = [ - "%s=%s" % (k, v) for k, v in list(params.items()) - ] + unordered_query = ["{}={}".format(k, v) for k, v in list(params.items())] query = query + unordered_query + ordered_query query = "&".join(query) - return urllib.parse.urlunsplit( - [scheme, netloc, u.path, query, u.fragment] - ) + return urllib.parse.urlunsplit([scheme, netloc, u.path, query, u.fragment]) def webreq(self, method, url, headers=None, path=None, **params): """ @@ -299,7 +290,7 @@ def push( @exception: BQCommError if the requests returns an error code and message """ - log.debug("POST %s req %s" % (url, headers)) + log.debug("POST {} req {}".format(url, headers)) try: # error checking r = self.request( @@ -312,9 +303,7 @@ def push( ) r.raise_for_status() except requests.exceptions.HTTPError: - log.exception( - "In push request: %s %s %s" % (method, url, r.content) - ) + log.exception("In push request: {} {} {}".format(method, url, r.content)) raise BQCommError(r) if path: @@ -325,7 +314,7 @@ def push( return r.content -class BQSession(object): +class BQSession: """ Top level Bisque communication object """ @@ -348,9 +337,7 @@ def _create_mex(self, user, moduleuri): mex = BQMex() mex.name = moduleuri or "script:%s" % " ".join(sys.argv) mex.status = "RUNNING" - self.mex = self.save( - mex, url=self.service_url("module_service", "mex") - ) + self.mex = self.save(mex, url=self.service_url("module_service", "mex")) if self.mex: mextoken = self.mex.resource_uniq self.c.authenticate_mex(mextoken, user) @@ -373,9 +360,7 @@ def _check_session(self): users = r.findall('./tag[@name="user"]') return len(users) > 0 - def init( - self, bisque_url, credentials=None, moduleuri=None, create_mex=False - ): + def init(self, bisque_url, credentials=None, moduleuri=None, create_mex=False): """Create session by connect to with bisque_url @param bisque_url: The bisque root or MEX url @@ -410,9 +395,7 @@ def init( ) return self - def init_local( - self, user, pwd, moduleuri=None, bisque_root=None, create_mex=True - ): + def init_local(self, user, pwd, moduleuri=None, bisque_root=None, create_mex=True): """ Initalizes a local session @@ -435,9 +418,7 @@ def init_local( self.c.authenticate_basic(user, pwd) self._load_services() if not self._check_session(): - log.error( - "Session failed to be created.. please check credentials" - ) + log.error("Session failed to be created.. please check credentials") return None self.mex = None @@ -473,9 +454,7 @@ def init_mex(self, mex_url, token, user=None, bisque_root=None): self.mex = self.load(mex_url, view="deep") return self - def init_cas( - self, user, pwd, moduleuri=None, bisque_root=None, create_mex=False - ): + def init_cas(self, user, pwd, moduleuri=None, bisque_root=None, create_mex=False): """Initalizes a cas session @param: user - a bisque user @@ -507,9 +486,7 @@ def init_cas( caslogin(self.c, bisque_root + "/auth_service/login", user, pwd) self._load_services() if not self._check_session(): - log.error( - "Session failed to be created.. please check credentials" - ) + log.error("Session failed to be created.. please check credentials") return None self.mex = None @@ -547,9 +524,7 @@ def close(self): def parameter(self, name): if self.mex is None: return None - return self.mex.xmltree.find( - 'tag[@name="inputs"]//tag[@name="%s"]' % name - ) + return self.mex.xmltree.find('tag[@name="inputs"]//tag[@name="%s"]' % name) def get_value_safe(self, v, t): try: @@ -601,11 +576,7 @@ def get_mex_inputs(self): """ def _xml2dict(e): - kids = { - key: e.attrib[key] - for key in e.attrib - if key in ["type", "value"] - } + kids = {key: e.attrib[key] for key in e.attrib if key in ["type", "value"]} if e.text: kids["value"] = e.text for k, g in itertools.groupby(e, lambda x: x.tag): @@ -641,9 +612,7 @@ def get_mex_execute_options(self): p = {} if self.mex is None: return p - for exop in self.mex.xmltree.iterfind( - 'tag[@name="execute_options"]/tag' - ): + for exop in self.mex.xmltree.iterfind('tag[@name="execute_options"]/tag'): p[exop.get("name")] = exop.get("value") return p @@ -695,7 +664,7 @@ def postxml(self, url, xml, path=None, method="POST", **params): if not isinstance(xml, str): xml = self.factory.to_string(xml) - log.debug("postxml %s content %s " % (url, xml)) + log.debug("postxml {} content {} ".format(url, xml)) url = self.c.prepare_url(url, **params) @@ -792,7 +761,7 @@ def service_url(self, service_type, path="", query=None): if root is None: raise BQApiError("Not a service type") if query: - path = "%s?%s" % (path, urllib.parse.urlencode(query)) + path = "{}?{}".format(path, urllib.parse.urlencode(query)) return urllib.parse.urljoin(root, path) def _load_services(self): @@ -888,9 +857,7 @@ def append_mex(mex, type_tup): assert len(hits) == 1 hits[0].extend(list(tg)) was_merged = True - log.debug( - "merged '%s' section in MEX", tg.get("name", "") - ) + log.debug("merged '%s' section in MEX", tg.get("name", "")) if not was_merged: mex.append(tg) @@ -903,9 +870,7 @@ def append_mex(mex, type_tup): # 'status' : status, # 'tag' : tags, # 'gobject': gobjects }} - content = self.postxml( - self.mex.uri, mex, view="deep" if reload else "short" - ) + content = self.postxml(self.mex.uri, mex, view="deep" if reload else "short") if reload and content is not None: self.mex = self.factory.from_string(content) return self.mex @@ -935,9 +900,7 @@ def finish_mex( merge=True, ) except BQCommError as ce: - log.error( - "Problem during finish mex %s" % ce.response.request.headers - ) + log.error("Problem during finish mex %s" % ce.response.request.headers) try: return self.update_mex( status="FAILED", @@ -967,9 +930,7 @@ def _begin_mex(self, moduleuri): ############################## # Module control ############################## - def run_modules( - self, module_list, pre_run=None, post_run=None, callback_fct=None - ): + def run_modules(self, module_list, pre_run=None, post_run=None, callback_fct=None): """Run one or more modules in parallel. :param module_list: List of modules to run @@ -998,9 +959,7 @@ def query(self, resource_type, **kw): tag_query=None, tag_order=None, offset=None, limit=None """ results = [] - queryurl = self.service_url( - "data_service", path=resource_type, query=kw - ) + queryurl = self.service_url("data_service", path=resource_type, query=kw) items = self.fetchxml(queryurl) for item in items: results.append(self.factory.from_etree(item)) @@ -1041,7 +1000,7 @@ def save(self, bqo, url=None, **kw): @return """ try: - _ = bqo # original + _ = bqo # original # Find an object (or parent with a valild uri) url = url or bqo.uri diff --git a/src/bqapi/exception.py b/src/bqapi/exception.py index 5600f84..849690a 100644 --- a/src/bqapi/exception.py +++ b/src/bqapi/exception.py @@ -22,8 +22,7 @@ def __init__(self, response): def __str__(self): content = ( - "%s...%s" - % (self.response.content[:64], self.response.content[-64:]) + "%s...%s" % (self.response.content[:64], self.response.content[-64:]) if len(self.response.content) > 64 else self.response.content ) diff --git a/src/bqapi/services.py b/src/bqapi/services.py index 45f94f6..d1dad98 100644 --- a/src/bqapi/services.py +++ b/src/bqapi/services.py @@ -1,16 +1,17 @@ +import json +import logging import os -# import urllib -# import urlparse - import random +import shutil import string -import logging import tempfile -import json -import shutil from six.moves import urllib +# import urllib +# import urlparse + + try: from lxml import etree except ImportError: @@ -22,9 +23,9 @@ logging.warn("pytables services not available") from requests_toolbelt import MultipartEncoder -from .util import normalize_unicode -from .exception import BQCommError +from .exception import BQCommError +from .util import normalize_unicode # DEFAULT_TIMEOUT=None DEFAULT_TIMEOUT = 60 * 60 # 1 hour @@ -99,9 +100,7 @@ def post(self, path=None, params=None, render=None, **kw): ) def put(self, path=None, params=None, render=None, **kw): - return self.request( - path=path, params=params, render=render, method="put", **kw - ) + return self.request(path=path, params=params, render=render, method="put", **kw) def delete(self, path=None, params=None, render=None, **kw): return self.request( @@ -159,9 +158,7 @@ def _resource_element( resource.set("name", os.path.basename(args_srcpath)) return resource - def path_link( - self, srcpath, alias=None, resource_type=None, tag_file=None - ): + def path_link(self, srcpath, alias=None, resource_type=None, tag_file=None): url = urllib.parse.urljoin( self.session.service_map["blob_service"], "paths/insert" ) @@ -271,20 +268,14 @@ def delete_member(self, dataset_uniq, resource_uniq, **kw): @return new dataset if success or None """ data = self.session.service("data_service") - dataset = data.fetch( - dataset_uniq, params={"view": "full"}, render="etree" - ) - members = dataset.xpath( - 'value[text()="%s"]' % data.construct(resource_uniq) - ) + dataset = data.fetch(dataset_uniq, params={"view": "full"}, render="etree") + members = dataset.xpath('value[text()="%s"]' % data.construct(resource_uniq)) for member in members: dataset.remove(member) if len(members): for val in dataset.iter("value"): _ = val.attrib.pop("index", 0) - return data.put( - dataset_uniq, data=etree.tostring(dataset), render="etree" - ) + return data.put(dataset_uniq, data=etree.tostring(dataset), render="etree") return None @@ -293,14 +284,10 @@ def execute(self, module_name, **module_parms): pass def register(self, engine_url): - return self.request( - path="register_engine", params={"engine_url": engine_url} - ) + return self.request(path="register_engine", params={"engine_url": engine_url}) def unregister(self, engine_url): - return self.request( - path="unregister_engine", params={"engine_url": engine_url} - ) + return self.request(path="unregister_engine", params={"engine_url": engine_url}) class TableProxy(BaseServiceProxy): @@ -317,9 +304,7 @@ def load_array(self, table_uniq, path, slices=[]): "%s;%s" % ( single_slice.start or "", - "" - if single_slice.stop is None - else single_slice.stop - 1, + "" if single_slice.stop is None else single_slice.stop - 1, ) ) elif isinstance(single_slice, int): @@ -394,9 +379,7 @@ def fetch_export(self, **kw): for key, val in list(kw.items()) if key in self.valid_param and val is not None } - response = self.fetch( - "stream", params=params, stream=kw.pop("stream", True) - ) + response = self.fetch("stream", params=params, stream=kw.pop("stream", True)) return response def fetch_export_local(self, localpath, stream=True, **kw): diff --git a/src/bqapi/tests/conftest.py b/src/bqapi/tests/conftest.py index e2ee066..ac5c80c 100644 --- a/src/bqapi/tests/conftest.py +++ b/src/bqapi/tests/conftest.py @@ -1,13 +1,16 @@ ## ## Add local fixtures here -import pytest from collections import OrderedDict, namedtuple +import pytest from bq.util.bunch import Bunch from bq.util.mkdir import _mkdir -from .util import fetch_file + from bqapi import BQServer +from .util import fetch_file + + @pytest.fixture(scope="module") def server(): return BQServer() diff --git a/src/bqapi/tests/test_bqapi.py b/src/bqapi/tests/test_bqapi.py index bdc494d..08d19b1 100644 --- a/src/bqapi/tests/test_bqapi.py +++ b/src/bqapi/tests/test_bqapi.py @@ -1,10 +1,9 @@ import pytest - from lxml import etree -from bqapi import BQSession -from bqapi.bqclass import BQFactory from tg import config +from bqapi import BQSession +from bqapi.bqclass import BQFactory pytestmark = pytest.mark.functional @@ -19,7 +18,7 @@ def test_load (session): #bq = BQSession() #bq.init_local (user, passwd, bisque_root = host, create_mex = False) x = session.load ('/data_service/image/?limit=10') - print "loading /data_service/images->", BQFactory.to_string((x)) + print "loading /data_service/images->", BQFactory.to_string(x) def test_load_pixels(session): diff --git a/src/bqapi/tests/test_bqclass.py b/src/bqapi/tests/test_bqclass.py index 893077d..3734441 100644 --- a/src/bqapi/tests/test_bqclass.py +++ b/src/bqapi/tests/test_bqclass.py @@ -1,6 +1,6 @@ import pytest - from lxml import etree + from bqapi.bqclass import BQFactory pytestmark = pytest.mark.unit diff --git a/src/bqapi/tests/test_bqfeature.py b/src/bqapi/tests/test_bqfeature.py index 4a03d8f..a86706d 100644 --- a/src/bqapi/tests/test_bqfeature.py +++ b/src/bqapi/tests/test_bqfeature.py @@ -1,32 +1,32 @@ import os -import numpy as np import urllib -from util import fetch_file -from lxml import etree -import ConfigParser +from collections import OrderedDict, namedtuple from datetime import datetime - -from collections import OrderedDict, namedtuple -import pytest +import ConfigParser import nose +import numpy as np +import pytest +from bq.util.mkdir import _mkdir +from lxml import etree from nose import with_setup +from util import fetch_file -from bq.util.mkdir import _mkdir -from bqapi import BQSession, BQServer -from bqapi.util import fetch_dataset +from bqapi import BQServer, BQSession +from bqapi.bqfeature import * from bqapi.comm import BQCommError from bqapi.util import * -from bqapi.bqfeature import * - +from bqapi.util import fetch_dataset - -TEST_PATH = 'tests_%s'%urllib.quote(datetime.now().strftime('%Y%m%d%H%M%S%f')) #set a test dir on the system so not too many repeats occur +TEST_PATH = "tests_%s" % urllib.quote( + datetime.now().strftime("%Y%m%d%H%M%S%f") +) # set a test dir on the system so not too many repeats occur pytestmark = pytest.mark.skip("Unported tests") -#pytestmark = pytest.mark.functional +# pytestmark = pytest.mark.functional + -#setup comm test +# setup comm test def setUp(): global results_location global store_local_location @@ -36,35 +36,38 @@ def setUp(): global FeatureResource config = ConfigParser.ConfigParser() - config.read('setup.cfg') - root = config.get('Host', 'root') or 'localhost:8080' - user = config.get('Host', 'user') or 'test' - pwd = config.get('Host', 'password') or 'test' - results_location = config.get('Store', 'results_location') or 'Results' + config.read("setup.cfg") + root = config.get("Host", "root") or "localhost:8080" + user = config.get("Host", "user") or "test" + pwd = config.get("Host", "password") or "test" + results_location = config.get("Store", "results_location") or "Results" _mkdir(results_location) - store_location = config.get('Store', 'location') or None - if store_location is None: raise NameError('Requre a store location to run test properly') + store_location = config.get("Store", "location") or None + if store_location is None: + raise NameError("Requre a store location to run test properly") - store_local_location = config.get('Store', 'local_location') or 'SampleData' - filename1 = config.get('Store','filename1') or None - if filename1 is None: raise NameError('Requre an image to run test properly') + store_local_location = config.get("Store", "local_location") or "SampleData" + filename1 = config.get("Store", "filename1") or None + if filename1 is None: + raise NameError("Requre an image to run test properly") file1_location = fetch_file(filename1, store_location, store_local_location) - FeatureResource = namedtuple('FeatureResource',['image','mask','gobject']) + FeatureResource = namedtuple("FeatureResource", ["image", "mask", "gobject"]) FeatureResource.__new__.__defaults__ = (None, None, None) - #start session + # start session bqsession = BQSession().init_local(user, pwd, bisque_root=root, create_mex=False) + def setup_bqfeature_fetch(): """ - uploads an image + uploads an image """ global resource_list - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element("resource", name="%s/%s" % (TEST_PATH, filename1)) content = bqsession.postblob(file1_location, xml=resource) - uniq = etree.XML(content)[0].attrib['resource_uniq'] - image_uri = '%s/image_service/image/%s'%(bqsession.bisque_root,uniq) + uniq = etree.XML(content)[0].attrib["resource_uniq"] + image_uri = "%s/image_service/image/%s" % (bqsession.bisque_root, uniq) resource_list = [FeatureResource(image=image_uri)] @@ -75,31 +78,32 @@ def teardown_bqfeature_fetch(): @with_setup(setup_bqfeature_fetch, teardown_bqfeature_fetch) def test_bqfeature_fetch_1(): """ - Test feature fetch and returning hdf5 file + Test feature fetch and returning hdf5 file """ - filename = 'bqfeature_fetch_1.h5' + filename = "bqfeature_fetch_1.h5" path = os.path.join(results_location, filename) - filename = Feature().fetch(bqsession, 'SimpleTestFeature', resource_list, path=path) + filename = Feature().fetch(bqsession, "SimpleTestFeature", resource_list, path=path) @with_setup(setup_bqfeature_fetch, teardown_bqfeature_fetch) def test_bqfeature_fetch_2(): """ - Test feature fetch and returning pytables object + Test feature fetch and returning pytables object """ - hdf5 = Feature().fetch(bqsession, 'SimpleTestFeature', resource_list) + hdf5 = Feature().fetch(bqsession, "SimpleTestFeature", resource_list) hdf5.close() os.remove(hdf5.filename) + def setup_bqfeature_fetchvector(): """ - uploads an image + uploads an image """ global resource_list - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element("resource", name="%s/%s" % (TEST_PATH, filename1)) content = bqsession.postblob(file1_location, xml=resource) - uniq = etree.XML(content)[0].attrib['resource_uniq'] - image_uri = '%s/image_service/image/%s'%(bqsession.bisque_root,uniq) + uniq = etree.XML(content)[0].attrib["resource_uniq"] + image_uri = "%s/image_service/image/%s" % (bqsession.bisque_root, uniq) resource_list = [FeatureResource(image=image_uri)] @@ -109,17 +113,26 @@ def teardown_bqfeature_fetchvector(): def test_bqfeature_fetchvector_1(): """ - Test fetch vector + Test fetch vector """ - feature_vector = Feature().fetch_vector(bqsession, 'SimpleTestFeature', resource_list) + feature_vector = Feature().fetch_vector( + bqsession, "SimpleTestFeature", resource_list + ) + def test_bqfeature_fetchvector_error(): """ - Test fetch vector on a resource that doesnt exist + Test fetch vector on a resource that doesnt exist """ try: - resource_list = [FeatureResource(image='%s/image_service/image/notaresource' % bqsession.bisque_root)] - feature_vector = Feature().fetch_vector(bqsession, 'SimpleTestFeature', resource_list) + resource_list = [ + FeatureResource( + image="%s/image_service/image/notaresource" % bqsession.bisque_root + ) + ] + feature_vector = Feature().fetch_vector( + bqsession, "SimpleTestFeature", resource_list + ) except FeatureError: assert True else: @@ -128,71 +141,78 @@ def test_bqfeature_fetchvector_error(): def setup_bqparallelfeature_fetch(): """ - uploads a list of images + uploads a list of images """ global resource_list resource_list = [] for _ in xrange(10): - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element("resource", name="%s/%s" % (TEST_PATH, filename1)) content = bqsession.postblob(file1_location, xml=resource) - uniq = etree.XML(content)[0].attrib['resource_uniq'] - resource_list.append(FeatureResource(image='%s/image_service/image/%s'%(bqsession.bisque_root,uniq))) + uniq = etree.XML(content)[0].attrib["resource_uniq"] + resource_list.append( + FeatureResource( + image="%s/image_service/image/%s" % (bqsession.bisque_root, uniq) + ) + ) def teardown_bqparallelfeature_fetch(): - """ - """ + """ """ pass @with_setup(setup_bqparallelfeature_fetch, teardown_bqparallelfeature_fetch) def test_bqparallelfeature_fetch_1(): """ - Test parallel feature fetch vector and returning pytables object + Test parallel feature fetch vector and returning pytables object """ - PF=ParallelFeature() - hdf5 = PF.fetch(bqsession, 'SimpleTestFeature', resource_list) + PF = ParallelFeature() + hdf5 = PF.fetch(bqsession, "SimpleTestFeature", resource_list) hdf5.close() os.remove(hdf5.filename) + @with_setup(setup_bqparallelfeature_fetch, teardown_bqparallelfeature_fetch) def test_bqparallelfeature_fetch_2(): """ - Test parallel feature fetch vector and return a file + Test parallel feature fetch vector and return a file """ - filename = 'bqparallelfeature_fetch_2.h5' + filename = "bqparallelfeature_fetch_2.h5" path = os.path.join(results_location, filename) - PF=ParallelFeature() + PF = ParallelFeature() PF.set_thread_num(2) PF.set_chunk_size(5) - filename = PF.fetch(bqsession, 'SimpleTestFeature', resource_list, path=path) + filename = PF.fetch(bqsession, "SimpleTestFeature", resource_list, path=path) def setup_bqparallelfeature_fetchvector(): """ - Uploads a list of images + Uploads a list of images """ global resource_list resource_list = [] for _ in xrange(10): - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element("resource", name="%s/%s" % (TEST_PATH, filename1)) content = bqsession.postblob(file1_location, xml=resource) - uniq = etree.XML(content)[0].attrib['resource_uniq'] - resource_list.append(FeatureResource(image='%s/image_service/image/%s'%(bqsession.bisque_root,uniq))) + uniq = etree.XML(content)[0].attrib["resource_uniq"] + resource_list.append( + FeatureResource( + image="%s/image_service/image/%s" % (bqsession.bisque_root, uniq) + ) + ) def teardown_bqparallelfeature_fetchvector(): - """ - """ + """ """ pass @with_setup(setup_bqparallelfeature_fetchvector, teardown_bqparallelfeature_fetchvector) def test_bqparallelfeature_fetchvector_1(): """ - Test parallel feature fetch vector + Test parallel feature fetch vector """ - PF=ParallelFeature() + PF = ParallelFeature() PF.set_thread_num(2) PF.set_chunk_size(5) - feature_vectors = PF.fetch_vector(bqsession, 'SimpleTestFeature', resource_list) + feature_vectors = PF.fetch_vector(bqsession, "SimpleTestFeature", resource_list) diff --git a/src/bqapi/tests/test_comm.py b/src/bqapi/tests/test_comm.py index 03f4460..dddcc9d 100644 --- a/src/bqapi/tests/test_comm.py +++ b/src/bqapi/tests/test_comm.py @@ -1,61 +1,62 @@ -import pytest - -from collections import OrderedDict, namedtuple import os -from lxml import etree +import time import urllib +from collections import OrderedDict, namedtuple from datetime import datetime -import time + +import pytest +from lxml import etree from bqapi import BQSession -TEST_PATH = 'tests_%s'%urllib.quote(datetime.now().strftime('%Y%m%d%H%M%S%f')) #set a test dir on the system so not too many repeats occur +TEST_PATH = "tests_%s" % urllib.quote( + datetime.now().strftime("%Y%m%d%H%M%S%f") +) # set a test dir on the system so not too many repeats occur # default mark is function.. may be overridden pytestmark = pytest.mark.functional + ############################# ### BQServer ############################# @pytest.mark.unit def test_prepare_url_1(server): - """ - """ - check_url = 'http://bisque.ece.ucsb.edu/image/00-123456789?remap=gray&format=tiff' - url = 'http://bisque.ece.ucsb.edu/image/00-123456789' - odict = OrderedDict([('remap','gray'),('format','tiff')]) + """ """ + check_url = "http://bisque.ece.ucsb.edu/image/00-123456789?remap=gray&format=tiff" + url = "http://bisque.ece.ucsb.edu/image/00-123456789" + odict = OrderedDict([("remap", "gray"), ("format", "tiff")]) url = server.prepare_url(url, odict=odict) assert url == check_url + @pytest.mark.unit def test_prepare_url_2(server): - """ - """ - check_url = 'http://bisque.ece.ucsb.edu/image/00-123456789?remap=gray&format=tiff' - url = 'http://bisque.ece.ucsb.edu/image/00-123456789' - url = server.prepare_url(url, remap='gray', format='tiff') + """ """ + check_url = "http://bisque.ece.ucsb.edu/image/00-123456789?remap=gray&format=tiff" + url = "http://bisque.ece.ucsb.edu/image/00-123456789" + url = server.prepare_url(url, remap="gray", format="tiff") assert url == check_url + @pytest.mark.unit def test_prepare_url_3(server): - """ - """ - check_url = 'http://bisque.ece.ucsb.edu/image/00-123456789?format=tiff&remap=gray' - url = 'http://bisque.ece.ucsb.edu/image/00-123456789' - odict = OrderedDict([('remap','gray')]) - url = server.prepare_url(url, odict=odict, format='tiff') + """ """ + check_url = "http://bisque.ece.ucsb.edu/image/00-123456789?format=tiff&remap=gray" + url = "http://bisque.ece.ucsb.edu/image/00-123456789" + odict = OrderedDict([("remap", "gray")]) + url = server.prepare_url(url, odict=odict, format="tiff") assert url == check_url - -#Test BQSession +# Test BQSession def test_open_session(config): """ - Test Initalizing a BQSession locally + Test Initalizing a BQSession locally """ - host = config.get ('host.root') - user = config.get ('host.user') - pwd = config.get ('host.password') + host = config.get("host.root") + user = config.get("host.user") + pwd = config.get("host.password") bqsession = BQSession().init_local(user, pwd, bisque_root=host, create_mex=False) bqsession.close() @@ -63,11 +64,11 @@ def test_open_session(config): def test_initalize_mex_locally(config): """ - Test initalizing a mex locally + Test initalizing a mex locally """ - host = config.get ('host.root') - user = config.get ('host.user') - pwd = config.get ('host.password') + host = config.get("host.root") + user = config.get("host.user") + pwd = config.get("host.password") bqsession = BQSession().init_local(user, pwd, bisque_root=host, create_mex=True) assert bqsession.mex.uri bqsession.close() @@ -75,11 +76,11 @@ def test_initalize_mex_locally(config): def test_initalize_session_From_mex(config): """ - Test initalizing a session from a mex + Test initalizing a session from a mex """ - host = config.get ('host.root') - user = config.get ('host.user') - pwd = config.get ('host.password') + host = config.get("host.root") + user = config.get("host.user") + pwd = config.get("host.password") bqsession = BQSession().init_local(user, pwd, bisque_root=host) mex_url = bqsession.mex.uri token = bqsession.mex.resource_uniq @@ -90,143 +91,151 @@ def test_initalize_session_From_mex(config): def test_fetchxml_1(session): """ - Test fetch xml + Test fetch xml """ - user = session.config.get ('host.user') - #bqsession = BQSession().init_local(user, pwd, bisque_root=root) - response_xml = session.fetchxml('/data_service/'+user) #fetches the user + user = session.config.get("host.user") + # bqsession = BQSession().init_local(user, pwd, bisque_root=root) + response_xml = session.fetchxml("/data_service/" + user) # fetches the user session.close() if not isinstance(response_xml, etree._Element): - assert False , 'Did not return XML!' + assert False, "Did not return XML!" + def test_fetchxml_2(session, stores): """ - Test fetch xml and save the document to disk + Test fetch xml and save the document to disk """ - user = session.config.get ('host.user') - filename = 'fetchxml_test_2.xml' - path = os.path.join(stores.results,filename) - path = session.fetchxml('/data_service/'+user, path=path) #fetches the user + user = session.config.get("host.user") + filename = "fetchxml_test_2.xml" + path = os.path.join(stores.results, filename) + path = session.fetchxml("/data_service/" + user, path=path) # fetches the user try: - with open(path,'r') as f: - etree.XML(f.read()) #check if xml was returned + with open(path, "r") as f: + etree.XML(f.read()) # check if xml was returned except etree.Error: - assert False , 'Did not return XML!' + assert False, "Did not return XML!" def test_postxml_1(session): """ - Test post xml + Test post xml """ - test_document =""" + test_document = """ """ - response_xml = session.postxml('/data_service/file', xml=test_document) + response_xml = session.postxml("/data_service/file", xml=test_document) if not isinstance(response_xml, etree._Element): - assert False ,'Did not return XML!' + assert False, "Did not return XML!" def test_postxml_2(session, stores): """ - Test post xml and save the document to disk + Test post xml and save the document to disk """ - test_document =""" + test_document = """ """ - filename = 'postxml_test_2.xml' - path = os.path.join(stores.results,filename) + filename = "postxml_test_2.xml" + path = os.path.join(stores.results, filename) - path = session.postxml('/data_service/file', test_document, path=path) + path = session.postxml("/data_service/file", test_document, path=path) try: - with open(path,'r') as f: - etree.XML(f.read()) #check if xml was returned + with open(path, "r") as f: + etree.XML(f.read()) # check if xml was returned except etree.Error: - assert False ,'Did not return XML!' + assert False, "Did not return XML!" def test_postxml_3(session): """ - Test post xml and read immediately + Test post xml and read immediately """ - test_document =""" + test_document = """ """ - response0_xml = session.postxml('/data_service/file', xml=test_document) - uri0 = response0_xml.get ('uri') + response0_xml = session.postxml("/data_service/file", xml=test_document) + uri0 = response0_xml.get("uri") response1_xml = session.fetchxml(uri0) - uri1 = response0_xml.get ('uri') - session.deletexml (url = uri0) + uri1 = response0_xml.get("uri") + session.deletexml(url=uri0) if not isinstance(response0_xml, etree._Element): - assert False , 'Did not return XML!' + assert False, "Did not return XML!" assert uri0 == uri1, "Posted and Fetched uri do not match" - - def test_fetchblob_1(): - """ - - """ + """ """ pass def test_postblob_1(session, stores): - """ Test post blob """ - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, stores.files[0].name)) + """Test post blob""" + resource = etree.Element( + "resource", name="%s/%s" % (TEST_PATH, stores.files[0].name) + ) content = session.postblob(stores.files[0].location, xml=resource) assert len(content), "No content returned" def test_postblob_2(session, stores): - """ Test post blob and save the returned document to disk """ - filename = 'postblob_test_2.xml' - path = os.path.join(stores.results,filename) - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, stores.files[0].name)) + """Test post blob and save the returned document to disk""" + filename = "postblob_test_2.xml" + path = os.path.join(stores.results, filename) + resource = etree.Element( + "resource", name="%s/%s" % (TEST_PATH, stores.files[0].name) + ) path = session.postblob(stores.files[0].location, xml=resource, path=path) try: - with open(path,'r') as f: - etree.XML(f.read()) #check if xml was returned + with open(path, "r") as f: + etree.XML(f.read()) # check if xml was returned except etree.Error: - assert False , 'Did not return XML!' + assert False, "Did not return XML!" + def test_postblob_3(session, stores): """ - Test post blob with xml attached + Test post blob with xml attached """ - test_document = """ + test_document = ( + """ - """%u'%s/%s'%(TEST_PATH, stores.files[0].name) + """ + % "%s/%s" + % (TEST_PATH, stores.files[0].name) + ) content = session.postblob(stores.files[0].location, xml=test_document) def test_run_mex(mexsession): """ - Test run mex + Test run mex """ session = mexsession mex_uri = session.mex.uri - session.update_mex(status="IN PROGRESS", tags = [], gobjects = [], children=[], reload=False) - response_xml = session.fetchxml(mex_uri) #check xml + session.update_mex( + status="IN PROGRESS", tags=[], gobjects=[], children=[], reload=False + ) + response_xml = session.fetchxml(mex_uri) # check xml session.finish_mex() - response_xml = session.fetchxml(mex_uri) #check xml - assert mex_uri == response_xml.get ('uri') + response_xml = session.fetchxml(mex_uri) # check xml + assert mex_uri == response_xml.get("uri") diff --git a/src/bqapi/tests/test_util.py b/src/bqapi/tests/test_util.py index b798a25..de48a82 100644 --- a/src/bqapi/tests/test_util.py +++ b/src/bqapi/tests/test_util.py @@ -1,18 +1,19 @@ -import pytest import os +from datetime import datetime + import numpy as np +import pytest +from bq.util.mkdir import _mkdir #import urllib from six.moves import urllib -from datetime import datetime - - -from bqapi import BQSession, BQServer -from bqapi.util import fetch_dataset -from bq.util.mkdir import _mkdir -from .util import fetch_file +from bqapi import BQServer, BQSession from bqapi.comm import BQCommError from bqapi.util import * +from bqapi.util import fetch_dataset + +from .util import fetch_file + try: from lxml import etree except ImportError: @@ -28,7 +29,7 @@ def image_uri(session, stores): """ uploads an image """ - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, stores.files[0].name)) + resource = etree.Element ('resource', name='%s/%s'%(TEST_PATH, stores.files[0].name)) content = bqsession.postblob(store.files[0].location, xml=resource) return etree.XML(content)[0].attrib['uri'] @@ -38,7 +39,7 @@ def setup_fetchimageplanes(): uploads an image """ global image_uri - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element ('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) image_uri = etree.XML(content)[0].attrib['uri'] @@ -53,7 +54,7 @@ def setup_fetchimagepixels(): uploads an image """ global image_uri - resource = etree.Element('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) image_uri = etree.XML(content)[0].attrib['uri'] @@ -68,7 +69,7 @@ def setup_fetchdataset(): global dataset_uri dataset = etree.Element('dataset', name='test') for _ in xrange(4): - resource = etree.Element('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) value=etree.SubElement(dataset,'value', type="object") value.text = etree.XML(content)[0].attrib['uri'] @@ -87,7 +88,7 @@ def setup_fetchDataset(): global dataset_uri dataset = etree.Element('dataset', name='test') for _ in xrange(4): - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element ('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) value=etree.SubElement(dataset,'value', type="object") value.text = etree.XML(content)[0].attrib['uri'] @@ -105,7 +106,7 @@ def setup_saveimagepixels(): uploads an image """ global image_uri - resource = etree.Element('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) image_uri = etree.XML(content)[0].attrib['uri'] @@ -119,7 +120,7 @@ def setup_fetchImage(): uploads an image """ global image_uri - resource = etree.Element ('resource', name=u'%s/%s'%(TEST_PATH, filename1)) + resource = etree.Element ('resource', name='%s/%s'%(TEST_PATH, filename1)) content = bqsession.postblob(stores.files[0].location, xml=resource) image_uri = etree.XML(content)[0].attrib['uri'] @@ -285,7 +286,7 @@ def test_saveimagepixels(): - """%u'%s/%s'%(TEST_PATH, filename1) + """%'%s/%s'%(TEST_PATH, filename1) #bqimage = fromXml(etree.XML(xmldoc)) bqimage = bqsession.factory.from_string (xmldoc) try: diff --git a/src/bqapi/tests/util.py b/src/bqapi/tests/util.py index e06b987..c7f622b 100644 --- a/src/bqapi/tests/util.py +++ b/src/bqapi/tests/util.py @@ -1,15 +1,17 @@ -from bq.util.mkdir import _mkdir +import os import posixpath import urllib -import os + +from bq.util.mkdir import _mkdir + def fetch_file(filename, url, dir): """ - @param filename: name of the file fetching from the store - @param url: url of the store - @param dir: the directory the file will be placed in - - @return the local path to the file + @param filename: name of the file fetching from the store + @param url: url of the store + @param dir: the directory the file will be placed in + + @return the local path to the file """ _mkdir(url) _mkdir(dir) @@ -17,4 +19,4 @@ def fetch_file(filename, url, dir): path = os.path.join(dir, filename) if not os.path.exists(path): urllib.urlretrieve(url, path) - return path \ No newline at end of file + return path diff --git a/src/bqapi/util.py b/src/bqapi/util.py index b5b1da9..097de8f 100644 --- a/src/bqapi/util.py +++ b/src/bqapi/util.py @@ -1,10 +1,10 @@ -import os -import shutil - # import urllib # import urlparse # import time import logging +import os +import shutil + from six.moves import urllib # from lxml import etree as ET @@ -58,9 +58,7 @@ def localpath2url(path): else: def url2localpath(url): - url = url.encode( - "utf-8" - ) # safegurd against un-encoded values in the DB + url = url.encode("utf-8") # safegurd against un-encoded values in the DB path = urllib.parse.urlparse(url).path return urllib.parse.unquote(path) @@ -69,6 +67,7 @@ def localpath2url(path): url = "file://%s" % url return url + ##################################################### diff --git a/src/bqapi_ucsb.egg-info/PKG-INFO b/src/bqapi_ucsb.egg-info/PKG-INFO index 8b1d5a6..7de7bf7 100644 --- a/src/bqapi_ucsb.egg-info/PKG-INFO +++ b/src/bqapi_ucsb.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: bqapi-ucsb -Version: 1.0.4 +Version: 1.0.6 Summary: Python API for interacting with BisQue Home-page: https://github.com/UCSB-VRL/bisqueUCSB Author: Bisque Team diff --git a/src/bqapi_ucsb.egg-info/SOURCES.txt b/src/bqapi_ucsb.egg-info/SOURCES.txt index 7781de3..974a73e 100644 --- a/src/bqapi_ucsb.egg-info/SOURCES.txt +++ b/src/bqapi_ucsb.egg-info/SOURCES.txt @@ -28,4 +28,4 @@ src/bqapi_ucsb.egg-info/PKG-INFO src/bqapi_ucsb.egg-info/SOURCES.txt src/bqapi_ucsb.egg-info/dependency_links.txt src/bqapi_ucsb.egg-info/requires.txt -src/bqapi_ucsb.egg-info/top_level.txt \ No newline at end of file +src/bqapi_ucsb.egg-info/top_level.txt diff --git a/src/bqapi_ucsb.egg-info/dependency_links.txt b/src/bqapi_ucsb.egg-info/dependency_links.txt index 8b13789..e69de29 100644 --- a/src/bqapi_ucsb.egg-info/dependency_links.txt +++ b/src/bqapi_ucsb.egg-info/dependency_links.txt @@ -1 +0,0 @@ -