관리-도구
편집 파일: spark_driver.cpython-37.pyc
B T a! � @ s� d dl mZ d dlmZ d dlmZ d dlmZ d dlm Z e rhd dl m Z d dl m Z d dlm Z mZ G d d � d e�Zd d � Zd d� Zdd� ZG dd� de�ZG dd� de�ZdS )� )�configure_scope)�Hub)� Integration)�capture_internal_exceptions)�MYPY)�Any)�Optional)�Event�Hintc @ s e Zd ZdZedd� �ZdS )�SparkIntegrationZsparkc C s t � d S )N)�patch_spark_context_init� r r �[/opt/alt/python37/lib/python3.7/site-packages/sentry_sdk/integrations/spark/spark_driver.py� setup_once s zSparkIntegration.setup_onceN)�__name__� __module__� __qualname__� identifier� staticmethodr r r r r r s r c C s6 ddl m} | j}|r2|�d|j� |�d|j� dS )z� Set properties in driver that propagate to worker processes, allowing for workers to have access to those properties. This allows worker integration to have access to app_name and application_id. r )� SparkContextZsentry_app_nameZsentry_application_idN)�pysparkr Z_active_spark_contextZsetLocalProperty�appName� applicationId)r Z spark_contextr r r �_set_app_properties s r c C s4 ddl m} | j}||� t� }| j�� �|� dS )zA Start java gateway server to add custom `SparkListener` r )� ensure_callback_server_startedN)Zpyspark.java_gatewayr Z_gateway�SentryListenerZ_jsc�scZaddSparkListener)r r ZgwZlistenerr r r �_start_sentry_listener( s r c s( ddl m} | j� � fdd�}|| _d S )Nr )r c sZ �� f|�|�}t j�t�d kr$|S t� � t� t� �}|j� fdd��}W d Q R X |S )Nc s t � �� tj�t�d kr | S | �di ��d� �� � | �di ��d� j�d�� | d �d� j�d�� | d �d� j�d �� | d �d � j�d �� | d �d � j � | d �d � j � | d �d� j � | d �d� j � | d �d� j � | �di ��d� j� W d Q R X | S )N�user�idZtagsz executor.idzspark.executor.idzspark-submit.deployModezspark.submit.deployModez driver.hostzspark.driver.hostz driver.portzspark.driver.portZ spark_versionZapp_nameZapplication_id�masterZ spark_home�extraZweb_url)r r �current�get_integrationr � setdefaultZ sparkUserZ_conf�get�versionr r r Z sparkHomeZuiWebUrl)�event�hint)�selfr r � process_eventG s( z[patch_spark_context_init.<locals>._sentry_patched_spark_context_init.<locals>.process_event)r r" r# r r r r Zadd_event_processor)r) �args�kwargs�initZscoper* )�spark_context_init)r) r �"_sentry_patched_spark_context_init; s zDpatch_spark_context_init.<locals>._sentry_patched_spark_context_init)r r Z_do_init)r r/ r )r. r r 5 s .r c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� ZG d1d2� d2�Zd3S )4� SparkListenerc C s d S )Nr )r) ZapplicationEndr r r �onApplicationEndm s z SparkListener.onApplicationEndc C s d S )Nr )r) ZapplicationStartr r r �onApplicationStartq s z SparkListener.onApplicationStartc C s d S )Nr )r) ZblockManagerAddedr r r �onBlockManagerAddedu s z!SparkListener.onBlockManagerAddedc C s d S )Nr )r) ZblockManagerRemovedr r r �onBlockManagerRemovedy s z#SparkListener.onBlockManagerRemovedc C s d S )Nr )r) Z blockUpdatedr r r �onBlockUpdated} s z SparkListener.onBlockUpdatedc C s d S )Nr )r) ZenvironmentUpdater r r �onEnvironmentUpdate� s z!SparkListener.onEnvironmentUpdatec C s d S )Nr )r) Z executorAddedr r r �onExecutorAdded� s z SparkListener.onExecutorAddedc C s d S )Nr )r) ZexecutorBlacklistedr r r �onExecutorBlacklisted� s z#SparkListener.onExecutorBlacklistedc C s d S )Nr )r) ZexecutorBlacklistedForStager r r � onExecutorBlacklistedForStage� s z+SparkListener.onExecutorBlacklistedForStagec C s d S )Nr )r) ZexecutorMetricsUpdater r r �onExecutorMetricsUpdate� s z%SparkListener.onExecutorMetricsUpdatec C s d S )Nr )r) ZexecutorRemovedr r r �onExecutorRemoved� s z SparkListener.onExecutorRemovedc C s d S )Nr )r) �jobEndr r r �onJobEnd� s zSparkListener.onJobEndc C s d S )Nr )r) �jobStartr r r � onJobStart� s zSparkListener.onJobStartc C s d S )Nr )r) ZnodeBlacklistedr r r �onNodeBlacklisted� s z SparkListener.onNodeBlacklistedc C s d S )Nr )r) ZnodeBlacklistedForStager r r �onNodeBlacklistedForStage� s z'SparkListener.onNodeBlacklistedForStagec C s d S )Nr )r) ZnodeUnblacklistedr r r �onNodeUnblacklisted� s z!SparkListener.onNodeUnblacklistedc C s d S )Nr )r) r' r r r � onOtherEvent� s zSparkListener.onOtherEventc C s d S )Nr )r) ZspeculativeTaskr r r �onSpeculativeTaskSubmitted� s z(SparkListener.onSpeculativeTaskSubmittedc C s d S )Nr )r) �stageCompletedr r r �onStageCompleted� s z SparkListener.onStageCompletedc C s d S )Nr )r) �stageSubmittedr r r �onStageSubmitted� s z SparkListener.onStageSubmittedc C s d S )Nr )r) ZtaskEndr r r � onTaskEnd� s zSparkListener.onTaskEndc C s d S )Nr )r) ZtaskGettingResultr r r �onTaskGettingResult� s z!SparkListener.onTaskGettingResultc C s d S )Nr )r) Z taskStartr r r � onTaskStart� s zSparkListener.onTaskStartc C s d S )Nr )r) Z unpersistRDDr r r �onUnpersistRDD� s z SparkListener.onUnpersistRDDc @ s e Zd ZdgZdS )zSparkListener.Javaz1org.apache.spark.scheduler.SparkListenerInterfaceN)r r r Z implementsr r r r �Java� s rM N) r r r r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r= r? r@ rA rB rC rD rF rH rI rJ rK rL rM r r r r r0 l s2 r0 c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r c C s t j| _d S )N)r r" �hub)r) r r r �__init__� s zSentryListener.__init__c C s( d� |�� �}| jjd|d� t� d S )NzJob {} Started�info)�level�message)�format�jobIdrN �add_breadcrumbr )r) r> rR r r r r? � s zSentryListener.onJobStartc C sd d}d}d|� � �� i}|� � �� dkr<d}d�|�� �}nd}d�|�� �}| jj|||d� d S ) N� �resultZ JobSucceededrP z Job {} Ended�warningz Job {} Failed)rQ rR �data)Z jobResult�toStringrS rT rN rU )r) r< rQ rR rY r r r r= � s zSentryListener.onJobEndc C sD |� � }d�|�� �}|�� |�� d�}| jjd||d� t� d S )NzStage {} Submitted)� attemptId�namerP )rQ rR rY )� stageInforS �stageIdr[ r\ rN rU r )r) rG � stage_inforR rY r r r rH � s z SentryListener.onStageSubmittedc C s� ddl m} |�� }d}d}|�� |�� d�}y&|�� �� |d< d�|�� �}d}W n& |k rz d�|�� �}d }Y nX | j j |||d � d S ) Nr )� Py4JJavaErrorrV )r[ r\ �reasonzStage {} FailedrX zStage {} CompletedrP )rQ rR rY ) Z py4j.protocolr` r] r[ r\ Z failureReasonr% rS r^ rN rU )r) rE r` r_ rR rQ rY r r r rF � s z SentryListener.onStageCompletedN)r r r rO r? r= rH rF r r r r r � s r N)� sentry_sdkr Zsentry_sdk.hubr Zsentry_sdk.integrationsr Zsentry_sdk.utilsr Zsentry_sdk._typesr �typingr r r r r r r r �objectr0 r r r r r �<module> s 7g