From GersteinInfo
parse_pmids.py
#!/usr/bin/python
import os, sys
from GoogleSpreadsheet import GoogleSpreadsheet
from datetime import datetime
master_spreadsheet_id = "******************"
worksheet_id = "od6"
master_spreadsheet = GoogleSpreadsheet(master_spreadsheet_id, worksheet_id)
ncbiquery = "/home/mpw6/new_papers/ncbiquery.txt"
ncbiFile = open(ncbiquery,'w')
def buildQuery(master_spreadsheet, ncbiFile):
start = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id="
pmids =
end = "&rettype=xml&retmode=file"
for row in master_spreadsheet:
if row['pmid']:
pmids += row['pmid'].lstrip('\) + ','
pmids = pmids[:-1]
out = start + pmids + end + '\n'
ncbiFile.write(out)
buildQuery(master_spreadsheet, ncbiFile)
PubmedHandler.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
from datetime import date
from xml.sax import ContentHandler
import xml.sax.saxutils
class PubmedHandler(ContentHandler):
def __init__(self,output_file = 'export.tab'):
self.file = file(output_file,'w')
self.sep = '\t'
self.data = {}
self.key =
self.pmid = False
self.authors = []
self.handling = False
self.elements = ['Initials', #multiple, name unique
'LastName', #multiple, name unique
'MedlineTA', #unique
'PMID', #unique
'OtherID', #unique
'MedlinePgn', #unique
'Volume', #unique
'Issue', #unique
'Year', #inside PubDate
'Month', #inside PubDate
'Day', #inside PubDate
'ArticleTitle', #unique
'CollectiveName',#unique
'ArticleId',
'PubDate',
]
self.file.write(self.get_header())
def __del__(self):
self.file.close()
def startElement(self,name,attrs):
if name == 'PubmedArticle':
self.data = {'PMID':}
self.authors = []
self.is_articledate = False
self.is_pubdate = False
self.pmid = 0
self.handling = False
elif name == 'PubDate':
self.is_pubdate = True
self.data['Year'] =
self.data['Month']=
self.data['Day'] =
elif name in self.elements:
if name == 'ArticleId':
if attrs.getValue('IdType') == 'pmc':
self.key = 'PMCID'
self.data['PMCID'] =
self.handling = True
else:
self.key = name
self.handling = True
if name == 'PMID':
self.pmid += 1
if name not in ['Year','Month','Day','PMID']:
self.data[name] =
def endElement(self,name):
if name == 'PubmedArticle':
self.write_to_file()
elif name == 'PubDate':
self.is_pubdate = False
elif name == 'Author': #merge author
if self.get_data('Initials'):
self.authors.append(self.get_data('Initials') + ' ' + self.get_data('LastName'))
if self.get_data('CollectiveName'):
self.authors.append(self.get_data('CollectiveName'))
self.data['CollectiveName'] =
self.handling = False
def characters(self,content):
if self.handling:
if self.key in ['Year','Month','Day']:
if self.is_pubdate:
self.data[self.key] += content
elif self.key == 'PMID':
if self.pmid == 1:
self.data[self.key] += content
else:
self.data[self.key] += xml.sax.saxutils.escape(content)
def write_to_file(self):
try:
self.file.write(self.get_row().encode('utf-8'))
except UnicodeEncodeError as anomerr:
print anomerr
print self.get_row()
return[]
def get_row(self):
return self.sep.join([', '.join(self.authors),
self.get_data('MedlineTA'),
self.get_data('PMID'),
self.get_data('MedlinePgn'),
self.get_data('Volume'),
self.get_data('Year'),
self.get_citation(),
self.get_data('ArticleTitle'),
self.get_data('PMCID')]) + '\n'
def get_citation(self):
citation =
if self.get_data('Year'):
citation += '(' + self.get_data('Year') + ').' + ' '
citation += '' + self.get_data('MedlineTA') + '' + ' '
date_str =
ref_str =
#build date string
if self.get_data('Year'):
date_str += self.get_data('Year')
if self.get_data('Month'):
date_str += ' ' + self.get_data('Month')
if self.get_int_data('Day'):
date_str += ' %d' % self.get_int_data('Day')
date_str = date_str.strip()
#build ref string
if self.get_data('Volume'):
ref_str += self.get_data('Volume')
#if self.get_data('Issue'):
# ref_str += '('+self.get_data('Issue')+')'
if self.get_data('MedlinePgn'):
ref_str += ':' + self.get_data('MedlinePgn')
ref_str = ref_str.strip()
#if date_str:
#citation += date_str
if ref_str:
#if date_str:
#citation += ';'
citation += ref_str
if citation[-1] != '.':
citation += '.'
return citation
def get_data(self,key):
if self.data.has_key(key):
return self.data[key]
return
def get_int_data(self,key):
val = self.get_data(key)
try:
val = int(val)
return val
except ValueError:
pass
return 0
def get_header(self):
return self.sep.join(['Authors','Journal','PMID','Pages','Volume','Year','Citation','Title','PMCID']) + '\n'
import.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
from PubmedHandler import PubmedHandler
from xml import sax
import sys
import os
def main(input_file,output_file) :
parser = sax.make_parser()
handler = PubmedHandler(output_file)
parser.setContentHandler(handler)
parser.parse(input_file)
if __name__ == '__main__':
input_file = 'NCBIData.xml'
output_file = 'export.tab'
if len(sys.argv) > 1:
input_file = sys.argv[1]
if len(sys.argv) > 2:
output_file = sys.argv[2]
main(input_file,output_file)
cmd = "sed -e 's/ä/\ä/g' -e 's/ç/\ç/g' -e 's/ü/\ü/g' -e 's/ó/\ó/g' -e 's/ö/\ö/g' -e 's/ş/\ş/g' -e 's/í/\í/g' -e 's/é/\é/g' -e 's/ô/\ô/g' -e 's/è/\è/g' export.tab > export_out.tab"
os.system(cmd)
