aljaz/dev-gpt
1
0
Fork 0
mirror of https://github.com/aljazceru/dev-gpt.git synced 2025-12-20 07:04:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add package

Browse Source
This commit is contained in:
Florian Hönicke
2023-04-04 18:32:28 +02:00
parent de5f9cc2a8
commit c93f73936c
6 changed files with 272 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
.github/manual-release.yml vendored Normal file
View File

@@ -0,0 +1,37 @@
name: Manual Release
on:
workflow_dispatch:
inputs:
release_reason:
description: 'Short reason for this manual release'
required: true
jobs:
regular-release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
token: ${{ secrets.JINA_DEV_BOT }}
fetch-depth: 100
- uses: actions/setup-python@v2
with:
python-version: 3.7
- run: |
git fetch --depth=1 origin +refs/tags/*:refs/tags/*
npm install git-release-notes
pip install twine wheel
./scripts/release.sh final "${{ github.event.inputs.release_reason }}" "${{github.actor}}"
env:
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }}
- if: failure()
run: echo "nothing to release"
- name: bumping master version
uses: ad-m/github-push-action@v0.6.0
with:
github_token: ${{ secrets.JINA_DEV_BOT }}
tags: true
branch: main

8
README.md
View File

@@ -5,6 +5,14 @@ This project streamlines the creation and deployment of microservices.
Simply describe your task using natural language, and the system will automatically build and deploy your microservice.
To ensure the executor accurately aligns with your intended task, you can also provide test scenarios.
# Quickstart
```bash
pip install gptdeploy
gptdeploy --description "Take a pdf file as input, and returns the text it contains." \
--test "Takes https://www2.deloitte.com/content/dam/Deloitte/de/Documents/about-deloitte/Deloitte-Unternehmensgeschichte.pdf and returns a string that is at least 100 characters long"
```
# Overview
The graphic below illustrates the process of creating a microservice and deploying it to the cloud.
```mermaid

142
main.py
View File

@@ -63,8 +63,8 @@ def wrap_content_in_code_block(executor_content, file_name, tag):
def create_executor(
executor_description,
test_scenario,
description,
test,
output_path,
executor_name,
package,
@@ -77,7 +77,7 @@ def create_executor(
print_colored('', '############# Executor #############', 'red')
user_query = (
general_guidelines()
+ executor_file_task(executor_name, executor_description, test_scenario, package)
+ executor_file_task(executor_name, description, test, package)
+ chain_of_thought_creation()
)
conversation = gpt.Conversation()
@@ -93,7 +93,7 @@ def create_executor(
user_query = (
general_guidelines()
+ wrap_content_in_code_block(executor_content, 'executor.py', 'python')
+ test_executor_file_task(executor_name, test_scenario)
+ test_executor_file_task(executor_name, test)
)
conversation = gpt.Conversation()
test_executor_content_raw = conversation.query(user_query)
@@ -170,7 +170,7 @@ def get_executor_path(output_path, package, version):
package_path = '_'.join(package)
return f'{output_path}/{package_path}/v{version}'
def debug_executor(output_path, package, executor_description, test_scenario):
def debug_executor(output_path, package, description, test):
MAX_DEBUGGING_ITERATIONS = 10
error_before = ''
for i in range(1, MAX_DEBUGGING_ITERATIONS):
@@ -185,9 +185,9 @@ def debug_executor(output_path, package, executor_description, test_scenario):
user_query = (
f"General rules: " + not_allowed()
+ 'Here is the description of the task the executor must solve:\n'
+ executor_description
+ description
+ '\n\nHere is the test scenario the executor must pass:\n'
+ test_scenario
+ test
+ 'Here are all the files I use:\n'
+ all_files_string
+ (('This is an error that is already fixed before:\n'
@@ -225,11 +225,11 @@ class MaxDebugTimeReachedException(BaseException):
pass
def generate_executor_name(executor_description):
def generate_executor_name(description):
conversation = gpt.Conversation()
user_query = f'''
Generate a name for the executor matching the description:
"{executor_description}"
"{description}"
The executor name must fulfill the following criteria:
- camel case
- start with a capital letter
@@ -246,50 +246,11 @@ PDFParserExecutor
name = extract_content_from_result(name_raw, 'name.txt')
return name
@click.command()
@click.option('--executor-description', required=True, help='Description of the executor.')
@click.option('--test-scenario', required=True, help='Test scenario for the executor.')
@click.option('--num_approaches', default=3, type=int, help='Number of num_approaches to use to fulfill the task (default: 3).')
@click.option('--output_path', default='executor', help='Path to the output folder (must be empty). ')
def main(
executor_description,
test_scenario,
num_approaches=3,
output_path='executor',
):
generated_name = generate_executor_name(executor_description)
executor_name = f'{generated_name}{random.randint(0, 1000_000)}'
packages = get_possible_packages(executor_description, num_approaches)
recreate_folder(output_path)
for package in packages:
try:
create_executor(executor_description, test_scenario, output_path, executor_name, package)
# executor_name = 'MicroChainExecutor790050'
executor_path = debug_executor(output_path, package, executor_description, test_scenario)
# print('Executor can be built locally, now we will push it to the cloud.')
# jina_cloud.push_executor(executor_path)
print('Deploy a jina flow')
host = jina_cloud.deploy_flow(executor_name, executor_path)
print(f'Flow is deployed create the playground for {host}')
create_playground(executor_name, executor_path, host)
except MaxDebugTimeReachedException:
print('Could not debug the executor.')
continue
print(
'Executor name:', executor_name, '\n',
'Executor path:', executor_path, '\n',
'Host:', host, '\n',
'Playground:', f'streamlit run {executor_path}/app.py', '\n',
)
break
def get_possible_packages(executor_description, threads):
def get_possible_packages(description, threads):
print_colored('', '############# What package to use? #############', 'red')
user_query = f'''
Here is the task description of the problme you need to solve:
"{executor_description}"
"{description}"
First, write down all the subtasks you need to solve which require python packages.
For each subtask:
Provide a list of 1 to 3 python packages you could use to solve the subtask. Prefer modern packages.
@@ -318,66 +279,109 @@ package2,package3,...
return packages
@click.command()
@click.option('--description', required=True, help='Description of the executor.')
@click.option('--test', required=True, help='Test scenario for the executor.')
@click.option('--num_approaches', default=3, type=int, help='Number of num_approaches to use to fulfill the task (default: 3).')
@click.option('--output_path', default='executor', help='Path to the output folder (must be empty). ')
def main(
description,
test,
num_approaches=3,
output_path='executor',
):
generated_name = generate_executor_name(description)
executor_name = f'{generated_name}{random.randint(0, 1000_000)}'
packages = get_possible_packages(description, num_approaches)
recreate_folder(output_path)
for package in packages:
try:
create_executor(description, test, output_path, executor_name, package)
# executor_name = 'MicroChainExecutor790050'
executor_path = debug_executor(output_path, package, description, test)
# print('Executor can be built locally, now we will push it to the cloud.')
# jina_cloud.push_executor(executor_path)
print('Deploy a jina flow')
host = jina_cloud.deploy_flow(executor_name, executor_path)
print(f'Flow is deployed create the playground for {host}')
create_playground(executor_name, executor_path, host)
except MaxDebugTimeReachedException:
print('Could not debug the executor.')
continue
print(
'Executor name:', executor_name, '\n',
'Executor path:', executor_path, '\n',
'Host:', host, '\n',
'Playground:', f'streamlit run {executor_path}/app.py', '\n',
)
break
if __name__ == '__main__':
main()
'''
python main.py --description "Input is a url of a website as input and classifies it as either individual or business." --test "Takes https://jina.ai/ as input and returns 'business'. Takes https://hanxiao.io/ as input and returns 'individual'."
'''
# accomplished tasks:
# main(
# executor_description="The executor takes a url of a website as input and classifies it as either individual or business.",
# test_scenario='Takes https://jina.ai/ as input and returns "business". Takes https://hanxiao.io/ as input and returns "individual". ',
# description="Input is a url of a website as input and classifies it as either individual or business.",
# test='Takes https://jina.ai/ as input and returns "business". Takes https://hanxiao.io/ as input and returns "individual". ',
# )
# needs to prove:
# ######## Level 1 task #########
# main(
# executor_description="The executor takes a pdf file as input, parses it and returns the text.",
# description="The executor takes a pdf file as input, parses it and returns the text.",
# input_modality='pdf',
# output_modality='text',
# test_scenario='Takes https://www2.deloitte.com/content/dam/Deloitte/de/Documents/about-deloitte/Deloitte-Unternehmensgeschichte.pdf and returns a string that is at least 100 characters long',
# test='Takes https://www2.deloitte.com/content/dam/Deloitte/de/Documents/about-deloitte/Deloitte-Unternehmensgeschichte.pdf and returns a string that is at least 100 characters long',
# )
# main(
# executor_description="The executor takes a url of a website as input and returns the logo of the website as an image.",
# test_scenario='Takes https://jina.ai/ as input and returns an svg image of the logo.',
# description="The executor takes a url of a website as input and returns the logo of the website as an image.",
# test='Takes https://jina.ai/ as input and returns an svg image of the logo.',
# )
# # # ######## Level 1 task #########
# main(
# executor_description="The executor takes a pdf file as input, parses it and returns the text.",
# description="The executor takes a pdf file as input, parses it and returns the text.",
# input_modality='pdf',
# output_modality='text',
# test_scenario='Takes https://www2.deloitte.com/content/dam/Deloitte/de/Documents/about-deloitte/Deloitte-Unternehmensgeschichte.pdf and returns a string that is at least 100 characters long',
# test='Takes https://www2.deloitte.com/content/dam/Deloitte/de/Documents/about-deloitte/Deloitte-Unternehmensgeschichte.pdf and returns a string that is at least 100 characters long',
# )
# ######## Level 2 task #########
# main(
# executor_description="OCR detector",
# description="OCR detector",
# input_modality='image',
# output_modality='text',
# test_scenario='Takes https://miro.medium.com/v2/resize:fit:1024/0*4ty0Adbdg4dsVBo3.png as input and returns a string that contains "Hello, world"',
# test='Takes https://miro.medium.com/v2/resize:fit:1024/0*4ty0Adbdg4dsVBo3.png as input and returns a string that contains "Hello, world"',
# )
# ######## Level 3 task #########
main(
executor_description="The executor takes an mp3 file as input and returns bpm and pitch in a json.",
test_scenario='Takes https://cdn.pixabay.com/download/audio/2023/02/28/audio_550d815fa5.mp3 as input and returns a json with bpm and pitch',
)
# main(
# description="The executor takes an mp3 file as input and returns bpm and pitch in a json.",
# test='Takes https://cdn.pixabay.com/download/audio/2023/02/28/audio_550d815fa5.mp3 as input and returns a json with bpm and pitch',
# )
######### Level 4 task #########
# main(
# executor_description="The executor takes 3D objects in obj format as input "
# "and outputs a 2D image projection of that object where the full object is shown. ",
# description="The executor takes 3D objects in obj format as input "
# "and outputs a 2D image rendering of that object where the full object is shown. ",
# input_modality='3d',
# output_modality='image',
# test_scenario='Test that 3d object from https://raw.githubusercontent.com/polygonjs/polygonjs-assets/master/models/wolf.obj '
# test='Test that 3d object from https://raw.githubusercontent.com/polygonjs/polygonjs-assets/master/models/wolf.obj '
# 'is put in and out comes a 2d rendering of it',
# )
# ######## Level 8 task #########
# main(
# executor_description="The executor takes an image as input and returns a list of bounding boxes of all animals in the image.",
# description="The executor takes an image as input and returns a list of bounding boxes of all animals in the image.",
# input_modality='blob',
# output_modality='json',
# test_scenario='Take the image from https://thumbs.dreamstime.com/b/dog-professor-red-bow-tie-glasses-white-background-isolated-dog-professor-glasses-197036807.jpg as input and assert that the list contains at least one bounding box. ',
# test='Take the image from https://thumbs.dreamstime.com/b/dog-professor-red-bow-tie-glasses-white-background-isolated-dog-professor-glasses-197036807.jpg as input and assert that the list contains at least one bounding box. ',
# )

3
requirements.txt
View File

@@ -1,2 +1,3 @@
jina==3.14.1
click==8.1.3
click==8.1.3
streamlit==1.20.0

34
setup.py Normal file
View File

@@ -0,0 +1,34 @@
from setuptools import setup, find_packages
def read_requirements():
with open('requirements.txt', 'r', encoding='utf-8') as f:
return [line.strip() for line in f.readlines() if not line.startswith('#')]
setup(
name='gptdeploy',
version='0.1.0',
description='Use natural language interface to create, deploy and update your microservice infrastructure.',
long_description=open('README.md', 'r', encoding='utf-8').read(),
long_description_content_type='text/markdown',
author='Florian Hönicke',
author_email='florian.hoenicke@jina.ai',
url='https://github.com/jina-ai/gptdeploy',
packages=find_packages(),
install_requires=read_requirements(),
entry_points={
'console_scripts': [
'gptdeploy=main:main',
],
},
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
],
)

118
src/scripts/release.sh Executable file
View File

@@ -0,0 +1,118 @@
#!/usr/bin/env bash
# Requirements
# brew install hub
# npm install -g git-release-notes
# pip install twine wheel
set -ex
INIT_FILE='now/__init__.py'
VER_TAG='__version__ = '
RELEASENOTE='./node_modules/.bin/git-release-notes'
function escape_slashes {
sed 's/\//\\\//g'
}
function update_ver_line {
local OLD_LINE_PATTERN=$1
local NEW_LINE=$2
local FILE=$3
local NEW=$(echo "${NEW_LINE}" | escape_slashes)
sed -i '/'"${OLD_LINE_PATTERN}"'/s/.*/'"${NEW}"'/' "${FILE}"
head -n10 ${FILE}
}
function clean_build {
rm -rf dist
rm -rf *.egg-info
rm -rf build
}
function pub_pypi {
# publish to pypi
clean_build
python setup.py sdist
twine upload dist/*
clean_build
}
function git_commit {
git config --local user.email "dev-bot@jina.ai"
git config --local user.name "Jina Dev Bot"
git tag "v$RELEASE_VER" -m "$(cat ./CHANGELOG.tmp)"
git add $INIT_FILE ./CHANGELOG.md
git commit -m "chore(version): the next version will be $NEXT_VER" -m "build($RELEASE_ACTOR): $RELEASE_REASON"
}
function make_release_note {
${RELEASENOTE} ${LAST_VER}..HEAD .github/release-template.ejs > ./CHANGELOG.tmp
head -n10 ./CHANGELOG.tmp
printf '\n%s\n\n%s\n%s\n\n%s\n\n%s\n\n' "$(cat ./CHANGELOG.md)" "<a name="release-note-${RELEASE_VER//\./-}"></a>" "## Release Note (\`${RELEASE_VER}\`)" "> Release time: $(date +'%Y-%m-%d %H:%M:%S')" "$(cat ./CHANGELOG.tmp)" > ./CHANGELOG.md
}
BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [[ "$BRANCH" != "main" ]]; then
printf "You are not at main branch, exit\n";
exit 1;
fi
LAST_UPDATE=`git show --no-notes --format=format:"%H" $BRANCH | head -n 1`
LAST_COMMIT=`git show --no-notes --format=format:"%H" origin/$BRANCH | head -n 1`
if [ $LAST_COMMIT != $LAST_UPDATE ]; then
printf "Your local $BRANCH is behind the remote master, exit\n"
exit 1;
fi
# release the current version
export RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2)
LAST_VER=$(git tag -l | sort -V | tail -n1)
printf "last version: \e[1;32m$LAST_VER\e[0m\n"
if [[ $1 == "final" ]]; then
printf "this will be a final release: \e[1;33m$RELEASE_VER\e[0m\n"
NEXT_VER=$(echo $RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')
printf "bump master version to: \e[1;32m$NEXT_VER\e[0m\n"
make_release_note
pub_pypi
VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\'
update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE"
RELEASE_REASON="$2"
RELEASE_ACTOR="$3"
git_commit
elif [[ $1 == 'rc' ]]; then
printf "this will be a release candidate: \e[1;33m$RELEASE_VER\e[0m\n"
DOT_RELEASE_VER=$(echo $RELEASE_VER | sed "s/rc/\./")
NEXT_VER=$(echo $DOT_RELEASE_VER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')
NEXT_VER=$(echo $NEXT_VER | sed "s/\.\([^.]*\)$/rc\1/")
printf "bump master version to: \e[1;32m$NEXT_VER\e[0m, this will be the next version\n"
make_release_note
pub_pypi
VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\'
update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE"
RELEASE_REASON="$2"
RELEASE_ACTOR="$3"
git_commit
else
# as a prerelease, pypi update only, no back commit etc.
COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count)
NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER
printf "this will be a developmental release: \e[1;33m$NEXT_VER\e[0m\n"
VER_TAG_NEXT=$VER_TAG\'${NEXT_VER}\'
update_ver_line "$VER_TAG" "$VER_TAG_NEXT" "$INIT_FILE"
pub_pypi
fi