john pfeiffer
  • Home
  • Categories
  • Tags
  • Archives

google app engine remote api console and scripted delete all stats

cd C:\Users\John\Documents\GitHub\myproject\AppEngine
python "C:\Program Files (x86)\Google\google_appengine\remote_api_shell.py" -s myproject.appspot.com
Email:  (the admin user for the appengine account)
Password:

From the console you can do various things like:

>>> from google.appengine.ext import db
>>> from models.Foo import Foo

>>> entries = db.Query( Foo ).fetch( 1000 )
>>> for item in entries:
>>>     print item.oid
>>>     item.delete()

>>> q = Foo.all( keys_only = True ).fetch( 1000 )   # FYI q = Foo.all( keys_only = True ) DOES NOT WORK
>>> db.delete( q )


> q = Blobs.all( keys_only=True )
> i = 0
> while True:
>   result = q.fetch( 1000 )
>   i += len( result )
>   if len( result ) < 1000:
>     break
>   cursor = q.cursor()
>   q.with_cursor( cursor )
>   print i
>   # db.delete( result )

NOTE: in the example above it is assumed you have created python package models with file Foo.py with class Foo(db.Model)

Hack method if you want to try and guess (binary search) , here I'm guessing that there are more than 2,000 items

# https://appengine.google.com/datastore/explorer?app_id=myapp&kind=Foo&viewby=kind&query=SELECT+*+FROM+Foo&limit=20&offset=2000

EXAMPLE APP.YAML

application: test4
version: 1
runtime: python27
api_version: 1
threadsafe: true

handlers:
- url: /favicon\.ico
  static_files: favicon.ico
  upload: favicon\.ico

- url: /css
  static_dir: assets/css

- url: /js
  static_dir: assets/js

- url: /images
  static_dir: assets/images

- url: /img
  static_dir: assets/img

- url: /.*
  script: main.app

builtins:
- remote_api: on


libraries:
- name: webapp2
  version: latest

- name: jinja2
  version: latest

- name: pycrypto
  version: latest

IF you write a python script that includes Google App Engine libraries in the python path

THEN you can automate certain things (though there's a delay in responsiveness =)

# -*- coding: utf-8 -*-
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.ext import db

class Foo( db.Model ):
    oid = db.StringProperty()
    created = db.DateTimeProperty( auto_now_add=True )


def auth_func():
    """  :returns a tuple of username and password
    """
    username = raw_input( 'Google App Engine Admin User: ' )
    password = raw_input( 'Password: ' )
    return ( username, password )


remote_api_stub.ConfigureRemoteApi( None, '/_ah/remote_api', auth_func, 'myproject.appspot.com' )

entries = db.Query( Foo ).fetch( 2 )   # get all of the items in one request, better than for item in query for .all()
count = 0
for item in entries:
    print item.oid
    count += 1
print count

you can also branch out into .put(), Query( ENTITY ).filter( 'name =', name ).order( 'desc' ) etc.

The Stats section in AdminConsole can also be accessed programmatically (updated once every 48 hours?)

https://developers.google.com/appengine/docs/python/datastore/stats


# -*- coding: utf-8 -*-
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.ext import db

def auth_func():
    """  :returns a tuple of username and password
    """
    username = raw_input( 'Google App Engine Admin User: ' )
#    password = getpass()   # PyCharm does not support getpass() , maybe ? c = msvcrt.getch()
    password = raw_input( 'Password: ' )
    print username, password
    return ( username, password )


class Blobs( db.Model ):
    oid = db.StringProperty()
    hashv = db.StringProperty()
    size = db.IntegerProperty()
    created = db.DateTimeProperty(auto_now_add=True)

def delete_all_blobs():
    """ the Blobs model is deprecated in favor of Blob
    """
    entries = db.Query( Blobs ).fetch( 1000 )
    count = 0
    for item in entries:
        item.delete()
    #    print item.hashv
        count += 1
        print '.',
    print 'deleted {} blob'.format( count )


class Commit( db.Model ):
    oid = db.StringProperty()
    operation = db.StringProperty( ) # - operation="add", "update", "delete", "move", "undelete"
    seq = db.IntegerProperty( )      # sequence_number
    nodeOid = db.StringProperty( )   # the node oid
    created = db.DateTimeProperty( auto_now_add = True )    # the creation date of the Commit Entry


def get_commits_after( start_after=0 ):
    print "COMMITS:"
    seqs = list()
    current = start_after
    while True:                 # TODO: do while
        commits = db.Query( Commit ).filter( 'seq >', current ).fetch( 1000 )
        if not commits:
            break
        for commit in commits:
            seqs.append( int( commit.seq ) )
            current = commit.seq          # TODO: do this more efficiently
        print seqs[-1]
    return seqs



SERVER = 'myproject.appspot.com'

if __name__ == '__main__':
    remote_api_stub.ConfigureRemoteApi( None, '/_ah/remote_api', auth_func, SERVER )

    # CHECK FOR A MISSING SEQ NUMBER IN A SERIES OF COMMITS
    START = 0
#    START = 12780
    seqs = get_commits_after( START )
    print "retrieved {} commits".format( len( seqs ) )

    for i in xrange( START, len( seqs) ):
        if i +1 != seqs[i]:
            print 'Mismatch between for loop counter {} and seq {}'.format( i, seqs[i] )

  • « google app engine urlfetch fix error No api proxy found for service urlfetch
  • memcache cas lock race condition »

Published

May 28, 2013

Category

python-appengine

~560 words

Tags

  • api 3
  • appengine 18
  • console 5
  • delete 7
  • engine 12
  • google 18
  • python 180
  • remote 15
  • scripted 1
  • stats 1