Compare commits
2 Commits
ff20228fa6
...
0240ff9f8e
| Author | SHA1 | Date | |
|---|---|---|---|
| 0240ff9f8e | |||
| 45e2e1eb00 |
@@ -23,6 +23,3 @@ class Auth:
|
||||
self.client.login(handle, password)
|
||||
|
||||
return self.client
|
||||
|
||||
def is_logged(self) -> bool:
|
||||
return bool(getattr(self.client, "me", None))
|
||||
|
||||
@@ -171,52 +171,3 @@ class Operation:
|
||||
context.logger.info(
|
||||
f"{self.operation_name}: {total_processed} items processed.")
|
||||
return total_processed
|
||||
|
||||
|
||||
def run_operation(
|
||||
strategy,
|
||||
operation_name: str,
|
||||
filter_fn: Optional[Callable[[Any], bool]] = None
|
||||
) -> int:
|
||||
context = OperationContext()
|
||||
progress = ProgressTracker(operation=operation_name)
|
||||
|
||||
context.logger.info(
|
||||
f"Starting {operation_name} with batch_size={context.batch_size}, delay={context.delay}s"
|
||||
)
|
||||
|
||||
cursor = None
|
||||
total_processed = 0
|
||||
batch_num = 0
|
||||
|
||||
while True:
|
||||
batch_num += 1
|
||||
response = strategy.fetch(context, cursor)
|
||||
items = strategy.extract_items(response)
|
||||
|
||||
if not items:
|
||||
break
|
||||
|
||||
progress.batch(batch_num, len(items))
|
||||
|
||||
for item in items:
|
||||
if filter_fn and not filter_fn(item):
|
||||
continue
|
||||
|
||||
try:
|
||||
strategy.process_item(item, context)
|
||||
total_processed += 1
|
||||
progress.update(1)
|
||||
except Exception as e:
|
||||
context.logger.error(f"Error processing item: {e}")
|
||||
|
||||
cursor = strategy.get_cursor(response)
|
||||
if not cursor:
|
||||
break
|
||||
|
||||
if context.delay > 0:
|
||||
time.sleep(context.delay)
|
||||
|
||||
context.logger.info(
|
||||
f"{operation_name}: {total_processed} items processed.")
|
||||
return total_processed
|
||||
|
||||
Reference in New Issue
Block a user