```python
class MongoCache:
db = None
def __init__(self):
if not hasattr(MongoCache, 'pool'):
MongoCache.create_instance()
@
staticmethod def create_instance():
client = MongoClient(config.MONGO_URL)
MongoCache.db = client['spider']
def create(self, table, unique_key, origin_data):
if self.exists(table, unique_key):
return None
summaries = {k: generator_summary(v) for (k, v) in origin_data.items()}
return self.db[table].insert({
'unique_key': unique_key,
'data': origin_data,
'summaries': summaries
})
def get(self, table, unique_key):
data = self.db[table].find_one({'unique_key': unique_key})
if data is None:
return None
return data['data']
def exists(self, table, unique_key):
data = self.db[table].find_one({'unique_key': unique_key})
return data is not None
def is_changed(self, table, unique_key, origin_data):
if not self.exists(table, unique_key):
return True
last_summaries = self.db[table].find_one({'unique_key': unique_key})['summaries']
for (k, v) in origin_data.items():
summary = generator_summary(v)
last_summary = last_summaries.get(k, None)
# print('{} -> {} | {} -> {}'.format(k, v, summary, last_summary))
if last_summary is None or last_summary != summary:
return True
return False
def change_fields(self, table, unique_key, origin_data):
if not self.exists(table, unique_key):
return origin_data
changes = {}
last_summaries = self.db[table].find_one({'unique_key': unique_key})['summaries']
for (k, v) in origin_data.items():
last_summary = last_summaries.get(k, None)
# print('{} -> {} | {} -> {}'.format(k, v, summary, last_summary))
if last_summary is None or last_summary != generator_summary(v):
changes[k] = v
return changes
def update(self, table, unique_key, origin_data):
if not self.exists(table, unique_key):
return origin_data
new_summaries = {k: generator_summary(v) for (k, v) in origin_data.items()}
self.db[table].update_one({'unique_key': unique_key},
{'$set': {'data': origin_data, 'summaries': new_summaries}})
return origin_data
```