Merge pull request #168 from developmentseed/maintenance

Maintenance

Author: drewbo

.circleci/config.yml

@@ -1,45 +1,65 @@
 version: 2
+common: &common
+ working_directory: ~/label-maker
+ steps:
+ - checkout
+ - run:
+ name: install dependencies
+ command: pip install tox codecov --user
+ - run:
+ name: install tippecanoe
+ command: .circleci/install_tippecanoe.sh
+ - run:
+ name: run tox
+ command: ~/.local/bin/tox
 jobs:
- build:
+ "python-3.6":
+ <<: *common
  docker:
- - image: circleci/python:3.6-stretch
+ - image: circleci/python:3.6.5
+ environment:
+ - TOXENV=py36
 
+ "python-3.7":
+ <<: *common
+ docker:
+ - image: circleci/python:3.7.2
+ environment:
+ - TOXENV=py37
+
+ deploy:
+ docker:
+ - image: circleci/python:3.7.2
+ environment:
+ - TOXENV=release
  working_directory: ~/label-maker
  steps:
  - checkout
  - run:
- name: Installing Dependencies
+ name: verify git tag vs. version
  command: |
- sudo apt-get update
- sudo apt-get install libgdal-dev
- sudo pip install numpy==1.13.3
- sudo pip install -r requirements.txt
- sudo pip install -r requirements-dev.txt
- git clone git@github.com:mapbox/tippecanoe.git
- cd tippecanoe
- sudo make -j
- sudo make install
-
+ VERSION=$(python setup.py --version)
+ if [ "$VERSION" = "$CIRCLE_TAG" ]; then exit 0; else exit 3; fi
  - run:
- name: Pylint
- command: pylint ~/label-maker/label_maker --rcfile=~/label-maker/.config/pylintrc
+ name: init .pypirc
+ command: |
+ echo -e "[pypi]" >> ~/.pypirc
+ echo -e "username = $PYPI_USER" >> ~/.pypirc
+ echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc
  - run:
- name: Unit Tests
- command: python -m unittest discover -v -s test/unit
+ name: install dependencies
+ command: pip install tox --user
  - run:
- name: Integration Tests
- command: |
- sudo pip install .
- python -m unittest discover -v -s test/integration
-
+ name: run tox
+ command: ~/.local/bin/tox
  - add_ssh_keys:
  fingerprints:
  - "79:16:39:74:e9:b3:39:52:87:2c:90:aa:ee:3c:09:13"
-
  - run:
  name: Deploy documentation
  command: |
  if [ "${CIRCLE_BRANCH}" == "${PRODUCTION_BRANCH}" ]; then
+ pip install -r requirements-dev.txt
  cd docs
  make html
  cd _build/html
@@ -56,3 +76,20 @@ jobs:
  else
  echo "Not the branch you're looking for, skipping documentation deploy"
  fi
+workflows:
+ version: 2
+ build_and_deploy:
+ jobs:
+ - "python-3.6"
+ - "python-3.7":
+ filters: # required since `deploy` has tag filters AND requires `build`
+ tags:
+ only: /.*/
+ - deploy:
+ requires:
+ - "python-3.7"
+ filters:
+ tags:
+ only: /^[0-9]+.*/
+ branches:
+ ignore: /.*/

.circleci/install_tippecanoe.sh

@@ -0,0 +1,5 @@
+sudo apt-get update -y && sudo apt-get install libgdal-dev
+git clone git@github.com:mapbox/tippecanoe.git
+cd tippecanoe
+sudo make -j
+sudo make install

.gitignore

@@ -14,3 +14,5 @@ stdout*
 /integration*
 .idea/
 docs/_build/
+
+.tox

label_maker/download.py

@@ -3,8 +3,27 @@
 from os import path
 import tempfile
 import gzip
-from homura import download
+import requests
+from tqdm import tqdm
 
+def download(url, path):
+ """Download url to target path"""
+ file_size = int(requests.head(url).headers["Content-Length"])
+ header = {"Range": "bytes=%s-%s" % (0, file_size)}
+ pbar = tqdm(
+ total=file_size,
+ unit='B',
+ unit_scale=True,
+ desc=url.split('/')[-1]
+ )
+ req = requests.get(url, headers=header, stream=True)
+ with(open(path, 'ab')) as f:
+ for chunk in req.iter_content(chunk_size=1024):
+ if chunk:
+ f.write(chunk)
+ pbar.update(1024)
+ pbar.close()
+ return file_size
 
 def download_mbtiles(dest_folder, country, **kwargs):
  """Download QA Tiles for the selected country.

label_maker/label.py

@@ -9,7 +9,6 @@
 
 import numpy as np
 import mapbox_vector_tile
-import pyproj
 from shapely.geometry import shape, mapping, Polygon
 from shapely.errors import TopologicalError
 from rasterio.features import rasterize
@@ -325,9 +324,3 @@ def _create_empty_label(ml_type, classes):
  elif ml_type == 'segmentation':
  return np.zeros((256, 256), dtype=np.int)
  return None
-
-# Use with 'transform' to project to EPSG:4326
-project = partial(
- pyproj.transform,
- pyproj.Proj(init='epsg:3857'),
- pyproj.Proj(init='epsg:4326'))

label_maker/utils.py

@@ -4,11 +4,14 @@
 from urllib.parse import urlparse, parse_qs
 
 from mercantile import bounds
-from pyproj import Proj, transform
 from PIL import Image
 import numpy as np
 import requests
 import rasterio
+from rasterio.crs import CRS
+from rasterio.warp import transform, transform_bounds
+
+WGS84_CRS = CRS.from_epsg(4326)
 
 def url(tile, imagery):
  """Return a tile url provided an imagery template and a tile"""
@@ -49,8 +52,6 @@ def get_tile_tif(tile, imagery, folder, kwargs):
  imagery_offset = kwargs.get('imagery_offset') or [0, 0]
  with rasterio.open(imagery) as src:
  x_res, y_res = src.transform[0], src.transform[4]
- p1 = Proj({'init': 'epsg:4326'})
- p2 = Proj(**src.crs)
 
  # offset our imagery in the "destination pixel" space
  offset_bound = dict()
@@ -63,8 +64,12 @@ def get_tile_tif(tile, imagery, folder, kwargs):
  offset_bound['south'] = bound.south + imagery_offset[1] * deg_per_pix_y
 
  # project tile boundaries from lat/lng to source CRS
- tile_ul_proj = transform(p1, p2, offset_bound['west'], offset_bound['north'])
- tile_lr_proj = transform(p1, p2, offset_bound['east'], offset_bound['south'])
+ x, y = transform(WGS84_CRS, src.crs, [offset_bound['west']], [offset_bound['north']])
+ tile_ul_proj = x[0], y[0]
+
+ x, y = transform(WGS84_CRS, src.crs, [offset_bound['east']], [offset_bound['south']])
+ tile_lr_proj = x[0], y[0]
+
  # get origin point from the TIF
  tif_ul_proj = (src.bounds.left, src.bounds.top)
 
@@ -106,16 +111,12 @@ def get_tile_wms(tile, imagery, folder, kwargs):
 
  # find our tile bounding box
  bound = bounds(*[int(t) for t in tile.split('-')])
- p1 = Proj({'init': 'epsg:4326'})
- p2 = Proj({'init': wms_srs})
+ xmin, ymin, xmax, ymax = transform_bounds(WGS84_CRS, CRS.from_string(wms_srs), *bound, densify_pts=21)
 
  # project the tile bounding box from lat/lng to WMS SRS
- tile_ll_proj = transform(p1, p2, bound.west, bound.south)
- tile_ur_proj = transform(p1, p2, bound.east, bound.north)
- if wms_version == '1.3.0':
- bbox = tile_ll_proj[::-1] + tile_ur_proj[::-1]
- else:
- bbox = tile_ll_proj + tile_ur_proj
+ bbox = (
+ [ymin, xmin, ymax, xmax] if wms_version == "1.3.0" else [xmin, ymin, xmax, ymax]
+ )
 
  # request the image with the transformed bounding box and save
  wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox]))
@@ -136,7 +137,7 @@ def is_tif(imagery):
  valid_tif = False
  else:
  valid_tif = True
- except rasterio._err.CPLE_HttpResponseError: #pylint: disable=protected-access
+ except rasterio.errors.RasterioIOError:
  # rasterio cannot open the path. this is the case for a
  # tile service
  valid_tif = False

requirements.txt

@@ -1,20 +1,12 @@
 Cerberus==1.1
 click==6.7
 geojson==2.3.0
-homura==0.1.5
-humanize==0.5.1
 mapbox-vector-tile==1.2.0
-mbutil==0.3.0
-mercantile==1.0.0
-numpy==1.13.3
-olefile==0.44
-Pillow==4.3.0
-protobuf==3.5.0.post1
-pyclipper==1.0.6
-pycurl==7.43.0.1
-pyproj==1.9.5.1
-rasterio[s3]==1.0a12
+mercantile>=1.0.3
+numpy>=1.18.4
+Pillow==7.1.2
+rasterio[s3]>=1.1
 requests>=2.20.0
 Shapely>=1.6.3
-six==1.10.0
 tilepie==0.2.1
+tqdm>=4.46.0

setup.py

@@ -8,12 +8,14 @@
 
 here = op.abspath(op.dirname(__file__))
 
+extra_reqs = {
+ "test": ["pytest", "pytest-cov"],
+ "dev": ["pytest", "pytest-cov", "pre-commit"],
+}
+
 # get the dependencies and installs
 with io.open(op.join(here, 'requirements.txt'), encoding='utf-8') as f:
- all_reqs = f.read().split('\n')
-
-install_requires = [x.strip() for x in all_reqs if 'git+' not in x]
-dependency_links = [x.strip().replace('git+', '') for x in all_reqs if 'git+' not in x]
+ install_requires = f.read().split('\n')
 
 # readme
 with open('README.md') as f:
@@ -39,7 +41,7 @@
  packages=find_packages(exclude=['docs', 'tests*']),
  include_package_data=True,
  install_requires=install_requires,
- dependency_links=dependency_links,
+ extras_require=extra_reqs,
  long_description=readme,
  long_description_content_type="text/markdown"
 )

test/integration/test_segmentation_labels.py

@@ -33,7 +33,7 @@ def test_cli(self):
  '62093-50164-17': 2400,
  '62094-50162-17': 21234,
  '62094-50164-17': 19146,
- '62094-50163-17': 21613,
+ '62094-50163-17': 21611,
  '62093-50163-17': 31568
  }
 

test/integration/test_segmentation_labels_sparse.py

@@ -32,7 +32,7 @@ def test_cli(self):
  '62093-50164-17': 2400,
  '62094-50162-17': 21234,
  '62094-50164-17': 19146,
- '62094-50163-17': 21613,
+ '62094-50163-17': 21611,
  '62093-50163-17': 31568
  }